params: release geth 1.10.0 stable

This updates the consensus tests to commit 31d6630 and
adds support for access list transactions in the test runner.

Co-authored-by: Martin Holst Swende <martin@swende.se>

.github/CONTRIBUTING.md

@@ -30,11 +30,11 @@ Please make sure your contributions adhere to our coding guidelines:
 Before you submit a feature request, please check and make sure that it isn't 
 possible through some other means. The JavaScript-enabled console is a powerful 
 feature in the right hands. Please check our 
-[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+[Geth documentation page](https://geth.ethereum.org/docs/) for more info
 and help.
 
 ## Configuration, dependencies, and tests
 
-Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
+Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide)
 for more details on configuring your environment, managing project dependencies
 and testing procedures.

.github/ISSUE_TEMPLATE/bug.md

@@ -1,8 +1,10 @@
-Hi there,
-
-Please note that this is an issue tracker reserved for bug reports and feature requests.
-
-For general questions please use [discord](https://discord.gg/nthXNEv) or the Ethereum stack exchange at https://ethereum.stackexchange.com.
+---
+name: Report a bug
+about: Something with go-ethereum is not working as expected
+title: ''
+labels: 'type:bug'
+assignees: ''
+---
 
 #### System information
 

.github/ISSUE_TEMPLATE/feature.md

@@ -0,0 +1,17 @@
+---
+name: Request a feature
+about: Report a missing feature - e.g. as a step before submitting a PR
+title: ''
+labels: 'type:feature'
+assignees: ''
+---
+
+# Rationale
+
+Why should this feature exist?
+What are the use-cases?
+
+# Implementation
+
+Do you have ideas regarding the implementation of this feature?
+Are you willing to implement this feature?

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,9 @@
+---
+name: Ask a question
+about: Something is unclear
+title: ''
+labels: 'type:docs'
+assignees: ''
+---
+
+This should only be used in very rare cases e.g. if you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [discord](https://discord.gg/nthXNEv) or the Ethereum stack exchange at https://ethereum.stackexchange.com.

.travis.yml

@@ -5,7 +5,7 @@ jobs:
   allow_failures:
     - stage: build
       os: osx
-      go: 1.14.x
+      go: 1.15.x
       env:
         - azure-osx
         - azure-ios
@@ -15,8 +15,8 @@ jobs:
     # This builder only tests code linters on latest version of Go
     - stage: lint
       os: linux
-      dist: xenial
-      go: 1.15.x
+      dist: bionic
+      go: 1.16.x
       env:
         - lint
       git:
@@ -24,75 +24,12 @@ jobs:
       script:
         - go run build/ci.go lint
 
-    - stage: build
-      os: linux
-      dist: xenial
-      go: 1.13.x
-      env:
-        - GO111MODULE=on
-      script:
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
-
-    - stage: build
-      os: linux
-      dist: xenial
-      go: 1.14.x
-      env:
-        - GO111MODULE=on
-      script:
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
-
-    # These are the latest Go versions.
-    - stage: build
-      os: linux
-      arch: amd64
-      dist: xenial
-      go: 1.15.x
-      env:
-        - GO111MODULE=on
-      script:
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
-
-    - stage: build
-      if: type = pull_request
-      os: linux
-      arch: arm64
-      dist: xenial
-      go: 1.15.x
-      env:
-        - GO111MODULE=on
-      script:
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
-
-    - stage: build
-      os: osx
-      osx_image: xcode11.3
-      go: 1.15.x
-      env:
-        - GO111MODULE=on
-      script:
-        - echo "Increase the maximum number of open file descriptors on macOS"
-        - NOFILE=20480
-        - sudo sysctl -w kern.maxfiles=$NOFILE
-        - sudo sysctl -w kern.maxfilesperproc=$NOFILE
-        - sudo launchctl limit maxfiles $NOFILE $NOFILE
-        - sudo launchctl limit maxfiles
-        - ulimit -S -n $NOFILE
-        - ulimit -n
-        - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
-
     # This builder does the Ubuntu PPA upload
     - stage: build
       if: type = push
       os: linux
-      dist: xenial
-      go: 1.15.x
+      dist: bionic
+      go: 1.16.x
       env:
         - ubuntu-ppa
         - GO111MODULE=on
@@ -109,15 +46,15 @@ jobs:
             - python-paramiko
       script:
         - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -goversion 1.15 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - stage: build
       if: type = push
       os: linux
-      dist: xenial
+      dist: bionic
       sudo: required
-      go: 1.15.x
+      go: 1.16.x
       env:
         - azure-linux
         - GO111MODULE=on
@@ -129,32 +66,32 @@ jobs:
             - gcc-multilib
       script:
         # Build for the primary platforms that Trusty can manage
-        - go run build/ci.go install
-        - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - go run build/ci.go install -arch 386
-        - go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        - go run build/ci.go install -dlgo
+        - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
+        - go run build/ci.go install -dlgo -arch 386
+        - go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
         # Switch over GCC to cross compilation (breaks 386, hence why do it here only)
         - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
         - sudo ln -s /usr/include/asm-generic /usr/include/asm
 
-        - GOARM=5 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
-        - GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - GOARM=6 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
-        - GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - GOARM=7 go run build/ci.go install -arch arm -cc arm-linux-gnueabihf-gcc
-        - GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - go run build/ci.go install -arch arm64 -cc aarch64-linux-gnu-gcc
-        - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        - GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+        - GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
+        - GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+        - GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
+        - GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc
+        - GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
+        - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
+        - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
     - stage: build
       if: type = push
       os: linux
-      dist: xenial
+      dist: bionic
       services:
         - docker
-      go: 1.15.x
+      go: 1.16.x
       env:
         - azure-linux-mips
         - GO111MODULE=on
@@ -163,38 +100,29 @@ jobs:
       script:
         - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v
         - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done
-        - go run build/ci.go archive -arch mips -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        - go run build/ci.go archive -arch mips -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
         - go run build/ci.go xgo --alltools -- --targets=linux/mipsle --ldflags '-extldflags "-static"' -v
         - for bin in build/bin/*-linux-mipsle; do mv -f "${bin}" "${bin/-linux-mipsle/}"; done
-        - go run build/ci.go archive -arch mipsle -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        - go run build/ci.go archive -arch mipsle -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
         - go run build/ci.go xgo --alltools -- --targets=linux/mips64 --ldflags '-extldflags "-static"' -v
         - for bin in build/bin/*-linux-mips64; do mv -f "${bin}" "${bin/-linux-mips64/}"; done
-        - go run build/ci.go archive -arch mips64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        - go run build/ci.go archive -arch mips64 -type tar -signer LINUX_SIGNING_KEY signify SIGNIFY_KEY -upload gethstore/builds
 
         - go run build/ci.go xgo --alltools -- --targets=linux/mips64le --ldflags '-extldflags "-static"' -v
         - for bin in build/bin/*-linux-mips64le; do mv -f "${bin}" "${bin/-linux-mips64le/}"; done
-        - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
     - stage: build
       if: type = push
       os: linux
-      dist: xenial
+      dist: bionic
       addons:
         apt:
           packages:
-            - oracle-java8-installer
-            - oracle-java8-set-default
-      language: android
-      android:
-        components:
-          - platform-tools
-          - tools
-          - android-15
-          - android-19
-          - android-24
+            - openjdk-8-jdk
       env:
         - azure-android
         - maven-android
@@ -202,25 +130,33 @@ jobs:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://dl.google.com/go/go1.15.linux-amd64.tar.gz | tar -xz
+        # Install Android and it's dependencies manually, Travis is stale
+        - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+        - curl https://dl.google.com/android/repository/commandlinetools-linux-6858069_latest.zip -o android.zip
+        - unzip -q android.zip -d $HOME/sdk && rm android.zip
+        - mv $HOME/sdk/cmdline-tools $HOME/sdk/latest && mkdir $HOME/sdk/cmdline-tools && mv $HOME/sdk/latest $HOME/sdk/cmdline-tools
+        - export PATH=$PATH:$HOME/sdk/cmdline-tools/latest/bin
+        - export ANDROID_HOME=$HOME/sdk
+
+        - yes | sdkmanager --licenses >/dev/null
+        - sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle"
+
+        # Install Go to allow building with
+        - curl https://dl.google.com/go/go1.16.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip -o android-ndk-r19b.zip
-        - unzip -q android-ndk-r19b.zip && rm android-ndk-r19b.zip
-        - mv android-ndk-r19b $ANDROID_HOME/ndk-bundle
-
         - mkdir -p $GOPATH/src/github.com/ethereum
         - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum
-        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
+        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -signify SIGNIFY_KEY  -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - stage: build
       if: type = push
       os: osx
-      go: 1.15.x
+      go: 1.16.x
       env:
         - azure-osx
         - azure-ios
@@ -229,8 +165,8 @@ jobs:
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
-        - go run build/ci.go install
-        - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
+        - go run build/ci.go install -dlgo
+        - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
         # Build the iOS framework and upload it to CocoaPods and Azure
         - gem uninstall cocoapods -a -x
@@ -245,14 +181,45 @@ jobs:
 
         # Workaround for https://github.com/golang/go/issues/23749
         - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
-        - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
+        - go run build/ci.go xcode -signer IOS_SIGNING_KEY -signify SIGNIFY_KEY -deploy trunk -upload gethstore/builds
+
+    # These builders run the tests
+    - stage: build
+      os: linux
+      arch: amd64
+      dist: bionic
+      go: 1.16.x
+      env:
+        - GO111MODULE=on
+      script:
+        - go run build/ci.go test -coverage $TEST_PACKAGES
+
+    - stage: build
+      if: type = pull_request
+      os: linux
+      arch: arm64
+      dist: bionic
+      go: 1.16.x
+      env:
+        - GO111MODULE=on
+      script:
+        - go run build/ci.go test -coverage $TEST_PACKAGES
+
+    - stage: build
+      os: linux
+      dist: bionic
+      go: 1.15.x
+      env:
+        - GO111MODULE=on
+      script:
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder does the Azure archive purges to avoid accumulating junk
     - stage: build
       if: type = cron
       os: linux
-      dist: xenial
-      go: 1.15.x
+      dist: bionic
+      go: 1.16.x
       env:
         - azure-purge
         - GO111MODULE=on

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.15-alpine as builder
+FROM golang:1.16-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers git
 

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.15-alpine as builder
+FROM golang:1.16-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers git
 

README.md

@@ -6,7 +6,7 @@ Official Golang implementation of the Ethereum protocol.
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
-[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
+[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum)
 [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
 
 Automated builds are available for stable releases and the unstable master branch. Binary
@@ -14,7 +14,7 @@ archives are published at https://geth.ethereum.org/downloads/.
 
 ## Building the source
 
-For prerequisites and detailed build instructions please read the [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum) on the wiki.
+For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth).
 
 Building `geth` requires both a Go (version 1.13 or later) and a C compiler. You can install
 them using your favourite package manager. Once the dependencies are installed, run
@@ -36,18 +36,18 @@ directory.
 
 |    Command    | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
 | :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-|  **`geth`**   | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options.          |
-|   `abigen`    | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
+|  **`geth`**   | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options.          |
+|   `abigen`    | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. |
 |  `bootnode`   | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks.                                                                                                                                                                                                                                                                 |
 |     `evm`     | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`).                                                                                                                                                                                                                                                                     |
-| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details.                                                                                                                                                                                                     |
-|   `rlpdump`   | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`).                                                                                                                                                                                                                                 |
+| `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://eth.wiki/json-rpc/API) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details.                                                                                                                                                                                                     |
+|   `rlpdump`   | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`).                                                                                                                                                                                                                                 |
 |   `puppeth`   | a CLI wizard that aids in creating a new Ethereum network.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 
 ## Running `geth`
 
 Going through all the possible command line flags is out of scope here (please consult our
-[CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options)),
+[CLI Wiki page](https://geth.ethereum.org/docs/interface/command-line-options)),
 but we've enumerated a few common parameter combos to get you up to speed quickly
 on how you can run your own `geth` instance.
 
@@ -66,9 +66,9 @@ This command will:
  * Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag),
    causing it to download more data in exchange for avoiding processing the entire history
    of the Ethereum network, which is very CPU intensive.
- * Start up `geth`'s built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
-   (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
-   as well as `geth`'s own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
+ * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console),
+   (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://web3js.readthedocs.io/en/)
+   as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server).
    This tool is optional and if you leave it out you can always attach to an already running
    `geth` instance with `geth attach`.
 
@@ -170,8 +170,8 @@ accessible from the outside.
 
 As a developer, sooner rather than later you'll want to start interacting with `geth` and the
 Ethereum network via your own programs and not manually through the console. To aid
-this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC)
-and [`geth` specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)).
+this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API)
+and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)).
 These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based
 platforms, and named pipes on Windows).
 
@@ -277,7 +277,7 @@ $ bootnode --genkey=boot.key
 $ bootnode --nodekey=boot.key
 ```
 
-With the bootnode online, it will display an [`enode` URL](https://github.com/ethereum/wiki/wiki/enode-url-format)
+With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format)
 that other nodes can use to connect to it and exchange peer information. Make sure to
 replace the displayed IP address information (most probably `[::]`) with your externally
 accessible IP to get the actual `enode` URL.
@@ -314,13 +314,13 @@ ones either). To start a `geth` instance for mining, run it with all your usual
 by:
 
 ```shell
-$ geth <usual-flags> --mine --miner.threads=1 --etherbase=0x0000000000000000000000000000000000000000
+$ geth <usual-flags> --mine --miner.threads=1 --miner.etherbase=0x0000000000000000000000000000000000000000
 ```
 
 Which will start mining blocks and transactions on a single CPU thread, crediting all
-proceedings to the account specified by `--etherbase`. You can further tune the mining
-by changing the default gas limit blocks converge to (`--targetgaslimit`) and the price
-transactions are accepted at (`--gasprice`).
+proceedings to the account specified by `--miner.etherbase`. You can further tune the mining
+by changing the default gas limit blocks converge to (`--miner.targetgaslimit`) and the price
+transactions are accepted at (`--miner.gasprice`).
 
 ## Contribution
 
@@ -344,7 +344,7 @@ Please make sure your contributions adhere to our coding guidelines:
  * Commit messages should be prefixed with the package(s) they modify.
    * E.g. "eth, rpc: make trace configs optional"
 
-Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
+Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide)
 for more details on configuring your environment, managing project dependencies, and
 testing procedures.
 

SECURITY.md

@@ -2,31 +2,29 @@
 
 ## Supported Versions
 
-Please see Releases. We recommend to use the most recent released version.  
+Please see [Releases](https://github.com/ethereum/go-ethereum/releases). We recommend using the [most recently released version](https://github.com/ethereum/go-ethereum/releases/latest).
 
 ## Audit reports
 
 Audit reports are published in the `docs` folder: https://github.com/ethereum/go-ethereum/tree/master/docs/audits 
 
-
 | Scope | Date | Report Link |
 | ------- | ------- | ----------- |
 | `geth` | 20170425 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2017-04-25_Geth-audit_Truesec.pdf) |
 | `clef` | 20180914 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2018-09-14_Clef-audit_NCC.pdf) |
 
-
-
 ## Reporting a Vulnerability
 
 **Please do not file a public ticket** mentioning the vulnerability.
 
-To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org.
+To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publically disclosed security vulnerabilities.
+
+Use the built-in `geth version-check` feature to check whether the software is affected by any known vulnerability. This command will fetch the latest [`vulnerabilities.json`](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json) file which contains known security vulnerabilities concerning `geth`, and cross-check the data against its own version number.
 
 The following key may be used to communicate sensitive information to developers.
 
 Fingerprint: `AE96 ED96 9E47 9B00 84F3 E17F E88D 3334 FA5F 6A0A`
 
-
 ```
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1

accounts/abi/abi_test.go

@@ -43,6 +43,7 @@ const jsondata = `
 	{ "type" : "function", "name" : "uint64[2]", "inputs" : [ { "name" : "inputs", "type" : "uint64[2]" } ] },
 	{ "type" : "function", "name" : "uint64[]", "inputs" : [ { "name" : "inputs", "type" : "uint64[]" } ] },
 	{ "type" : "function", "name" : "int8", "inputs" : [ { "name" : "inputs", "type" : "int8" } ] },
+	{ "type" : "function", "name" : "bytes32", "inputs" : [ { "name" : "inputs", "type" : "bytes32" } ] },
 	{ "type" : "function", "name" : "foo", "inputs" : [ { "name" : "inputs", "type" : "uint32" } ] },
 	{ "type" : "function", "name" : "bar", "inputs" : [ { "name" : "inputs", "type" : "uint32" }, { "name" : "string", "type" : "uint16" } ] },
 	{ "type" : "function", "name" : "slice", "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] },
@@ -68,6 +69,7 @@ var (
 	String, _     = NewType("string", "", nil)
 	Bool, _       = NewType("bool", "", nil)
 	Bytes, _      = NewType("bytes", "", nil)
+	Bytes32, _    = NewType("bytes32", "", nil)
 	Address, _    = NewType("address", "", nil)
 	Uint64Arr, _  = NewType("uint64[]", "", nil)
 	AddressArr, _ = NewType("address[]", "", nil)
@@ -98,6 +100,7 @@ var methods = map[string]Method{
 	"uint64[]":            NewMethod("uint64[]", "uint64[]", Function, "", false, false, []Argument{{"inputs", Uint64Arr, false}}, nil),
 	"uint64[2]":           NewMethod("uint64[2]", "uint64[2]", Function, "", false, false, []Argument{{"inputs", Uint64Arr2, false}}, nil),
 	"int8":                NewMethod("int8", "int8", Function, "", false, false, []Argument{{"inputs", Int8, false}}, nil),
+	"bytes32":             NewMethod("bytes32", "bytes32", Function, "", false, false, []Argument{{"inputs", Bytes32, false}}, nil),
 	"foo":                 NewMethod("foo", "foo", Function, "", false, false, []Argument{{"inputs", Uint32, false}}, nil),
 	"bar":                 NewMethod("bar", "bar", Function, "", false, false, []Argument{{"inputs", Uint32, false}, {"string", Uint16, false}}, nil),
 	"slice":               NewMethod("slice", "slice", Function, "", false, false, []Argument{{"inputs", Uint32Arr2, false}}, nil),

accounts/abi/bind/auth.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/external"
@@ -28,11 +29,21 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
 )
 
+// ErrNoChainID is returned whenever the user failed to specify a chain id.
+var ErrNoChainID = errors.New("no chain id specified")
+
+// ErrNotAuthorized is returned when an account is not properly unlocked.
+var ErrNotAuthorized = errors.New("not authorized to sign this account")
+
 // NewTransactor is a utility method to easily create a transaction signer from
 // an encrypted json key stream and the associated passphrase.
+//
+// Deprecated: Use NewTransactorWithChainID instead.
 func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
+	log.Warn("WARNING: NewTransactor has been deprecated in favour of NewTransactorWithChainID")
 	json, err := ioutil.ReadAll(keyin)
 	if err != nil {
 		return nil, err
@@ -45,13 +56,17 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 }
 
 // NewKeyStoreTransactor is a utility method to easily create a transaction signer from
-// a decrypted key from a keystore.
+// an decrypted key from a keystore.
+//
+// Deprecated: Use NewKeyStoreTransactorWithChainID instead.
 func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) {
+	log.Warn("WARNING: NewKeyStoreTransactor has been deprecated in favour of NewTransactorWithChainID")
+	signer := types.HomesteadSigner{}
 	return &TransactOpts{
 		From: account.Address,
-		Signer: func(signer types.Signer, address common.Address, tx *types.Transaction) (*types.Transaction, error) {
+		Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) {
 			if address != account.Address {
-				return nil, errors.New("not authorized to sign this account")
+				return nil, ErrNotAuthorized
 			}
 			signature, err := keystore.SignHash(account, signer.Hash(tx).Bytes())
 			if err != nil {
@@ -64,13 +79,17 @@ func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account
 
 // NewKeyedTransactor is a utility method to easily create a transaction signer
 // from a single private key.
+//
+// Deprecated: Use NewKeyedTransactorWithChainID instead.
 func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts {
+	log.Warn("WARNING: NewKeyedTransactor has been deprecated in favour of NewKeyedTransactorWithChainID")
 	keyAddr := crypto.PubkeyToAddress(key.PublicKey)
+	signer := types.HomesteadSigner{}
 	return &TransactOpts{
 		From: keyAddr,
-		Signer: func(signer types.Signer, address common.Address, tx *types.Transaction) (*types.Transaction, error) {
+		Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) {
 			if address != keyAddr {
-				return nil, errors.New("not authorized to sign this account")
+				return nil, ErrNotAuthorized
 			}
 			signature, err := crypto.Sign(signer.Hash(tx).Bytes(), key)
 			if err != nil {
@@ -81,14 +100,73 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts {
 	}
 }
 
+// NewTransactorWithChainID is a utility method to easily create a transaction signer from
+// an encrypted json key stream and the associated passphrase.
+func NewTransactorWithChainID(keyin io.Reader, passphrase string, chainID *big.Int) (*TransactOpts, error) {
+	json, err := ioutil.ReadAll(keyin)
+	if err != nil {
+		return nil, err
+	}
+	key, err := keystore.DecryptKey(json, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	return NewKeyedTransactorWithChainID(key.PrivateKey, chainID)
+}
+
+// NewKeyStoreTransactorWithChainID is a utility method to easily create a transaction signer from
+// an decrypted key from a keystore.
+func NewKeyStoreTransactorWithChainID(keystore *keystore.KeyStore, account accounts.Account, chainID *big.Int) (*TransactOpts, error) {
+	if chainID == nil {
+		return nil, ErrNoChainID
+	}
+	signer := types.LatestSignerForChainID(chainID)
+	return &TransactOpts{
+		From: account.Address,
+		Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) {
+			if address != account.Address {
+				return nil, ErrNotAuthorized
+			}
+			signature, err := keystore.SignHash(account, signer.Hash(tx).Bytes())
+			if err != nil {
+				return nil, err
+			}
+			return tx.WithSignature(signer, signature)
+		},
+	}, nil
+}
+
+// NewKeyedTransactorWithChainID is a utility method to easily create a transaction signer
+// from a single private key.
+func NewKeyedTransactorWithChainID(key *ecdsa.PrivateKey, chainID *big.Int) (*TransactOpts, error) {
+	keyAddr := crypto.PubkeyToAddress(key.PublicKey)
+	if chainID == nil {
+		return nil, ErrNoChainID
+	}
+	signer := types.LatestSignerForChainID(chainID)
+	return &TransactOpts{
+		From: keyAddr,
+		Signer: func(address common.Address, tx *types.Transaction) (*types.Transaction, error) {
+			if address != keyAddr {
+				return nil, ErrNotAuthorized
+			}
+			signature, err := crypto.Sign(signer.Hash(tx).Bytes(), key)
+			if err != nil {
+				return nil, err
+			}
+			return tx.WithSignature(signer, signature)
+		},
+	}, nil
+}
+
 // NewClefTransactor is a utility method to easily create a transaction signer
 // with a clef backend.
 func NewClefTransactor(clef *external.ExternalSigner, account accounts.Account) *TransactOpts {
 	return &TransactOpts{
 		From: account.Address,
-		Signer: func(signer types.Signer, address common.Address, transaction *types.Transaction) (*types.Transaction, error) {
+		Signer: func(address common.Address, transaction *types.Transaction) (*types.Transaction, error) {
 			if address != account.Address {
-				return nil, errors.New("not authorized to sign this account")
+				return nil, ErrNotAuthorized
 			}
 			return clef.SignTx(account, transaction, nil) // Clef enforces its own chain id
 		},

accounts/abi/bind/backends/simulated.go

@@ -74,6 +74,7 @@ type SimulatedBackend struct {
 
 // NewSimulatedBackendWithDatabase creates a new binding backend based on the given database
 // and uses a simulated blockchain for testing purposes.
+// A simulated backend always uses chainID 1337.
 func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
@@ -91,6 +92,7 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis
 
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
+// A simulated backend always uses chainID 1337.
 func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
 	return NewSimulatedBackendWithDatabase(rawdb.NewMemoryDatabase(), alloc, gasLimit)
 }
@@ -123,10 +125,9 @@ func (b *SimulatedBackend) Rollback() {
 
 func (b *SimulatedBackend) rollback() {
 	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {})
-	stateDB, _ := b.blockchain.State()
 
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil)
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), b.blockchain.StateCache(), nil)
 }
 
 // stateByBlockNumber retrieves a state by a given blocknumber.
@@ -479,7 +480,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 		b.pendingState.RevertToSnapshot(snapshot)
 
 		if err != nil {
-			if err == core.ErrIntrinsicGas {
+			if errors.Is(err, core.ErrIntrinsicGas) {
 				return true, nil, nil // Special case, raise gas limit
 			}
 			return true, nil, err // Bail out
@@ -542,10 +543,11 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	// Execute the call.
 	msg := callMsg{call}
 
-	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil)
+	txContext := core.NewEVMTxContext(msg)
+	evmContext := core.NewEVMBlockContext(block.Header(), b.blockchain, nil)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
-	vmEnv := vm.NewEVM(evmContext, stateDB, b.config, vm.Config{})
+	vmEnv := vm.NewEVM(evmContext, txContext, stateDB, b.config, vm.Config{})
 	gasPool := new(core.GasPool).AddGas(math.MaxUint64)
 
 	return core.NewStateTransition(vmEnv, msg, gasPool).TransitionDb()
@@ -557,7 +559,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	sender, err := types.Sender(types.NewEIP155Signer(b.config.ChainID), tx)
+	// Check transaction validity.
+	block := b.blockchain.CurrentBlock()
+	signer := types.MakeSigner(b.blockchain.Config(), block.Number())
+	sender, err := types.Sender(signer, tx)
 	if err != nil {
 		panic(fmt.Errorf("invalid transaction: %v", err))
 	}
@@ -566,7 +571,8 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		panic(fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce))
 	}
 
-	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
+	// Include tx in chain.
+	blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
 		for _, tx := range b.pendingBlock.Transactions() {
 			block.AddTxWithChain(b.blockchain, tx)
 		}
@@ -705,14 +711,15 @@ type callMsg struct {
 	ethereum.CallMsg
 }
 
-func (m callMsg) From() common.Address { return m.CallMsg.From }
-func (m callMsg) Nonce() uint64        { return 0 }
-func (m callMsg) CheckNonce() bool     { return false }
-func (m callMsg) To() *common.Address  { return m.CallMsg.To }
-func (m callMsg) GasPrice() *big.Int   { return m.CallMsg.GasPrice }
-func (m callMsg) Gas() uint64          { return m.CallMsg.Gas }
-func (m callMsg) Value() *big.Int      { return m.CallMsg.Value }
-func (m callMsg) Data() []byte         { return m.CallMsg.Data }
+func (m callMsg) From() common.Address         { return m.CallMsg.From }
+func (m callMsg) Nonce() uint64                { return 0 }
+func (m callMsg) CheckNonce() bool             { return false }
+func (m callMsg) To() *common.Address          { return m.CallMsg.To }
+func (m callMsg) GasPrice() *big.Int           { return m.CallMsg.GasPrice }
+func (m callMsg) Gas() uint64                  { return m.CallMsg.Gas }
+func (m callMsg) Value() *big.Int              { return m.CallMsg.Value }
+func (m callMsg) Data() []byte                 { return m.CallMsg.Data }
+func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList }
 
 // filterBackend implements filters.Backend to support filtering for logs without
 // taking bloom-bits acceleration structures into account.

accounts/abi/bind/backends/simulated_test.go

@@ -39,7 +39,7 @@ import (
 func TestSimulatedBackend(t *testing.T) {
 	var gasLimit uint64 = 8000029
 	key, _ := crypto.GenerateKey() // nolint: gosec
-	auth := bind.NewKeyedTransactor(key)
+	auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 	genAlloc := make(core.GenesisAlloc)
 	genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)}
 
@@ -411,7 +411,7 @@ func TestSimulatedBackend_EstimateGas(t *testing.T) {
 
 	key, _ := crypto.GenerateKey()
 	addr := crypto.PubkeyToAddress(key.PublicKey)
-	opts := bind.NewKeyedTransactor(key)
+	opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 	sim := NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(params.Ether)}}, 10000000)
 	defer sim.Close()
@@ -888,7 +888,7 @@ func TestSimulatedBackend_PendingCodeAt(t *testing.T) {
 	if err != nil {
 		t.Errorf("could not get code at test addr: %v", err)
 	}
-	auth := bind.NewKeyedTransactor(testKey)
+	auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337))
 	contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim)
 	if err != nil {
 		t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract)
@@ -924,7 +924,7 @@ func TestSimulatedBackend_CodeAt(t *testing.T) {
 	if err != nil {
 		t.Errorf("could not get code at test addr: %v", err)
 	}
-	auth := bind.NewKeyedTransactor(testKey)
+	auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337))
 	contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim)
 	if err != nil {
 		t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract)
@@ -956,7 +956,7 @@ func TestSimulatedBackend_PendingAndCallContract(t *testing.T) {
 	if err != nil {
 		t.Errorf("could not get code at test addr: %v", err)
 	}
-	contractAuth := bind.NewKeyedTransactor(testKey)
+	contractAuth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337))
 	addr, _, _, err := bind.DeployContract(contractAuth, parsed, common.FromHex(abiBin), sim)
 	if err != nil {
 		t.Errorf("could not deploy contract: %v", err)
@@ -1043,7 +1043,7 @@ func TestSimulatedBackend_CallContractRevert(t *testing.T) {
 	if err != nil {
 		t.Errorf("could not get code at test addr: %v", err)
 	}
-	contractAuth := bind.NewKeyedTransactor(testKey)
+	contractAuth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337))
 	addr, _, _, err := bind.DeployContract(contractAuth, parsed, common.FromHex(reverterBin), sim)
 	if err != nil {
 		t.Errorf("could not deploy contract: %v", err)

accounts/abi/bind/base.go

@@ -32,7 +32,7 @@ import (
 
 // SignerFn is a signer function callback when a contract requires a method to
 // sign the transaction before submission.
-type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Transaction, error)
+type SignerFn func(common.Address, *types.Transaction) (*types.Transaction, error)
 
 // CallOpts is the collection of options to fine tune a contract call request.
 type CallOpts struct {
@@ -152,7 +152,10 @@ func (c *BoundContract) Call(opts *CallOpts, results *[]interface{}, method stri
 		}
 	} else {
 		output, err = c.caller.CallContract(ctx, msg, opts.BlockNumber)
-		if err == nil && len(output) == 0 {
+		if err != nil {
+			return err
+		}
+		if len(output) == 0 {
 			// Make sure we have a contract to operate on, and bail out otherwise.
 			if code, err = c.caller.CodeAt(ctx, c.address, opts.BlockNumber); err != nil {
 				return err
@@ -253,7 +256,7 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 	if opts.Signer == nil {
 		return nil, errors.New("no signer to authorize the transaction with")
 	}
-	signedTx, err := opts.Signer(types.HomesteadSigner{}, opts.From, rawTx)
+	signedTx, err := opts.Signer(opts.From, rawTx)
 	if err != nil {
 		return nil, err
 	}

accounts/abi/bind/bind_test.go

@@ -296,7 +296,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -351,7 +351,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -397,7 +397,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -455,7 +455,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -503,7 +503,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -529,6 +529,70 @@ var bindTests = []struct {
 		nil,
 		nil,
 	},
+	// Tests that structs are correctly unpacked
+	{
+
+		`Structs`,
+		`
+		pragma solidity ^0.6.5;
+			pragma experimental ABIEncoderV2;
+			contract Structs {
+				struct A {
+					bytes32 B;
+				}
+				
+				function F() public view returns (A[] memory a, uint256[] memory c, bool[] memory d) {
+					A[] memory a = new A[](2);
+					a[0].B = bytes32(uint256(1234) << 96);
+					uint256[] memory c;
+					bool[] memory d;
+					return (a, c, d);
+				}
+			
+				function G() public view returns (A[] memory a) {
+					A[] memory a = new A[](2);
+					a[0].B = bytes32(uint256(1234) << 96);
+					return a;
+				}
+			}
+		`,
+		[]string{`608060405234801561001057600080fd5b50610278806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806328811f591461003b5780636fecb6231461005b575b600080fd5b610043610070565b604051610052939291906101a0565b60405180910390f35b6100636100d6565b6040516100529190610186565b604080516002808252606082810190935282918291829190816020015b610095610131565b81526020019060019003908161008d575050805190915061026960611b9082906000906100be57fe5b60209081029190910101515293606093508392509050565b6040805160028082526060828101909352829190816020015b6100f7610131565b8152602001906001900390816100ef575050805190915061026960611b90829060009061012057fe5b602090810291909101015152905090565b60408051602081019091526000815290565b815260200190565b6000815180845260208085019450808401835b8381101561017b578151518752958201959082019060010161015e565b509495945050505050565b600060208252610199602083018461014b565b9392505050565b6000606082526101b3606083018661014b565b6020838203818501528186516101c98185610239565b91508288019350845b818110156101f3576101e5838651610143565b9484019492506001016101d2565b505084810360408601528551808252908201925081860190845b8181101561022b57825115158552938301939183019160010161020d565b509298975050505050505050565b9081526020019056fea2646970667358221220eb85327e285def14230424c52893aebecec1e387a50bb6b75fc4fdbed647f45f64736f6c63430006050033`},
+		[]string{`[{"inputs":[],"name":"F","outputs":[{"components":[{"internalType":"bytes32","name":"B","type":"bytes32"}],"internalType":"structStructs.A[]","name":"a","type":"tuple[]"},{"internalType":"uint256[]","name":"c","type":"uint256[]"},{"internalType":"bool[]","name":"d","type":"bool[]"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"G","outputs":[{"components":[{"internalType":"bytes32","name":"B","type":"bytes32"}],"internalType":"structStructs.A[]","name":"a","type":"tuple[]"}],"stateMutability":"view","type":"function"}]`},
+		`
+			"math/big"
+
+			"github.com/ethereum/go-ethereum/accounts/abi/bind"
+			"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+			"github.com/ethereum/go-ethereum/core"
+			"github.com/ethereum/go-ethereum/crypto"
+		`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
+		
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
+			defer sim.Close()
+		
+			// Deploy a structs method invoker contract and execute its default method
+			_, _, structs, err := DeployStructs(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy defaulter contract: %v", err)
+			}
+			sim.Commit()
+			opts := bind.CallOpts{}
+			if _, err := structs.F(&opts); err != nil {
+				t.Fatalf("Failed to invoke F method: %v", err)
+			}
+			if _, err := structs.G(&opts); err != nil {
+				t.Fatalf("Failed to invoke G method: %v", err)
+			}
+		`,
+		nil,
+		nil,
+		nil,
+		nil,
+	},
 	// Tests that non-existent contracts are reported as such (though only simulator test)
 	{
 		`NonExistent`,
@@ -569,6 +633,45 @@ var bindTests = []struct {
 		nil,
 		nil,
 	},
+	{
+		`NonExistentStruct`,
+		`
+			contract NonExistentStruct {
+				function Struct() public view returns(uint256 a, uint256 b) {
+					return (10, 10);
+				}
+			}
+		`,
+		[]string{`6080604052348015600f57600080fd5b5060888061001e6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063d5f6622514602d575b600080fd5b6033604c565b6040805192835260208301919091528051918290030190f35b600a809156fea264697066735822beefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeefbeef64736f6c6343decafe0033`},
+		[]string{`[{"inputs":[],"name":"Struct","outputs":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"stateMutability":"pure","type":"function"}]`},
+		`
+			"github.com/ethereum/go-ethereum/accounts/abi/bind"
+			"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+			"github.com/ethereum/go-ethereum/common"
+			"github.com/ethereum/go-ethereum/core"
+		`,
+		`
+			// Create a simulator and wrap a non-deployed contract
+
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{}, uint64(10000000000))
+			defer sim.Close()
+
+			nonexistent, err := NewNonExistentStruct(common.Address{}, sim)
+			if err != nil {
+				t.Fatalf("Failed to access non-existent contract: %v", err)
+			}
+			// Ensure that contract calls fail with the appropriate error
+			if res, err := nonexistent.Struct(nil); err == nil {
+				t.Fatalf("Call succeeded on non-existent contract: %v", res)
+			} else if (err != bind.ErrNoCode) {
+				t.Fatalf("Error mismatch: have %v, want %v", err, bind.ErrNoCode)
+			}
+		`,
+		nil,
+		nil,
+		nil,
+		nil,
+	},
 	// Tests that gas estimation works for contracts with weird gas mechanics too.
 	{
 		`FunkyGasPattern`,
@@ -598,7 +701,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -648,7 +751,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -723,7 +826,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -817,7 +920,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -1007,7 +1110,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -1142,7 +1245,7 @@ var bindTests = []struct {
 
 		`
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -1284,7 +1387,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -1350,7 +1453,7 @@ var bindTests = []struct {
 		`
 		// Initialize test accounts
 		key, _ := crypto.GenerateKey()
-		auth := bind.NewKeyedTransactor(key)
+		auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 		sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 		defer sim.Close()
 
@@ -1444,7 +1547,7 @@ var bindTests = []struct {
 		sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, 10000000)
 		defer sim.Close()
 
-		transactOpts := bind.NewKeyedTransactor(key)
+		transactOpts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 		_, _, _, err := DeployIdentifierCollision(transactOpts, sim)
 		if err != nil {
 			t.Fatalf("failed to deploy contract: %v", err)
@@ -1506,7 +1609,7 @@ var bindTests = []struct {
 		sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, 10000000)
 		defer sim.Close()
 
-		transactOpts := bind.NewKeyedTransactor(key)
+		transactOpts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 		_, _, c1, err := DeployContractOne(transactOpts, sim)
 		if err != nil {
 			t.Fatal("Failed to deploy contract")
@@ -1563,7 +1666,7 @@ var bindTests = []struct {
 		`
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
-			auth := bind.NewKeyedTransactor(key)
+			auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000)
 			defer sim.Close()
@@ -1601,11 +1704,7 @@ var bindTests = []struct {
 		contract NewFallbacks {
 			event Fallback(bytes data);
 			fallback() external {
-				bytes memory data;
-				assembly {
-					calldatacopy(data, 0, calldatasize())
-				}
-				emit Fallback(data);
+				emit Fallback(msg.data);
 			}
 	
 			event Received(address addr, uint value);
@@ -1614,7 +1713,7 @@ var bindTests = []struct {
 			}
 		}
 	   `,
-		[]string{"60806040523480156100115760006000fd5b50610017565b61016e806100266000396000f3fe60806040526004361061000d575b36610081575b7f88a5966d370b9919b20f3e2c13ff65706f196a4e32cc2c12bf57088f885258743334604051808373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020018281526020019250505060405180910390a15b005b34801561008e5760006000fd5b505b606036600082377f9043988963722edecc2099c75b0af0ff76af14ffca42ed6bce059a20a2a9f986816040518080602001828103825283818151815260200191508051906020019080838360005b838110156100fa5780820151818401525b6020810190506100de565b50505050905090810190601f1680156101275780820380516001836020036101000a031916815260200191505b509250505060405180910390a1505b00fea26469706673582212205643ca37f40c2b352dc541f42e9e6720de065de756324b7fcc9fb1d67eda4a7d64736f6c63430006040033"},
+		[]string{"6080604052348015600f57600080fd5b506101078061001f6000396000f3fe608060405236605f577f88a5966d370b9919b20f3e2c13ff65706f196a4e32cc2c12bf57088f885258743334604051808373ffffffffffffffffffffffffffffffffffffffff1681526020018281526020019250505060405180910390a1005b348015606a57600080fd5b507f9043988963722edecc2099c75b0af0ff76af14ffca42ed6bce059a20a2a9f98660003660405180806020018281038252848482818152602001925080828437600081840152601f19601f820116905080830192505050935050505060405180910390a100fea26469706673582212201f994dcfbc53bf610b19176f9a361eafa77b447fd9c796fa2c615dfd0aaf3b8b64736f6c634300060c0033"},
 		[]string{`[{"anonymous":false,"inputs":[{"indexed":false,"internalType":"bytes","name":"data","type":"bytes"}],"name":"Fallback","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"addr","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Received","type":"event"},{"stateMutability":"nonpayable","type":"fallback"},{"stateMutability":"payable","type":"receive"}]`},
 		`
 			"bytes"
@@ -1632,7 +1731,7 @@ var bindTests = []struct {
 			sim := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, 1000000)
 			defer sim.Close()
 	
-			opts := bind.NewKeyedTransactor(key)
+			opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
 			_, _, c, err := DeployNewFallbacks(opts, sim)
 			if err != nil {
 				t.Fatalf("Failed to deploy contract: %v", err)
@@ -1662,6 +1761,7 @@ var bindTests = []struct {
 			}
 	
 			// Test fallback function
+			gotEvent = false
 			opts.Value = nil
 			calldata := []byte{0x01, 0x02, 0x03}
 			c.Fallback(opts, calldata)
@@ -1746,11 +1846,16 @@ func TestGolangBindings(t *testing.T) {
 		t.Fatalf("failed to convert binding test to modules: %v\n%s", err, out)
 	}
 	pwd, _ := os.Getwd()
-	replacer := exec.Command(gocmd, "mod", "edit", "-replace", "github.com/ethereum/go-ethereum="+filepath.Join(pwd, "..", "..", "..")) // Repo root
+	replacer := exec.Command(gocmd, "mod", "edit", "-x", "-require", "github.com/ethereum/go-ethereum@v0.0.0", "-replace", "github.com/ethereum/go-ethereum="+filepath.Join(pwd, "..", "..", "..")) // Repo root
 	replacer.Dir = pkg
 	if out, err := replacer.CombinedOutput(); err != nil {
 		t.Fatalf("failed to replace binding test dependency to current source tree: %v\n%s", err, out)
 	}
+	tidier := exec.Command(gocmd, "mod", "tidy")
+	tidier.Dir = pkg
+	if out, err := tidier.CombinedOutput(); err != nil {
+		t.Fatalf("failed to tidy Go module file: %v\n%s", err, out)
+	}
 	// Test the entire package and report any failures
 	cmd := exec.Command(gocmd, "test", "-v", "-count", "1")
 	cmd.Dir = pkg

accounts/abi/bind/template.go

@@ -304,8 +304,11 @@ var (
 			err := _{{$contract.Type}}.contract.Call(opts, &out, "{{.Original.Name}}" {{range .Normalized.Inputs}}, {{.Name}}{{end}})
 			{{if .Structured}}
 			outstruct := new(struct{ {{range .Normalized.Outputs}} {{.Name}} {{bindtype .Type $structs}}; {{end}} })
+			if err != nil {
+				return *outstruct, err
+			}
 			{{range $i, $t := .Normalized.Outputs}} 
-			outstruct.{{.Name}} = out[{{$i}}].({{bindtype .Type $structs}}){{end}}
+			outstruct.{{.Name}} = *abi.ConvertType(out[{{$i}}], new({{bindtype .Type $structs}})).(*{{bindtype .Type $structs}}){{end}}
 
 			return *outstruct, err
 			{{else}}
@@ -541,6 +544,7 @@ var (
 			if err := _{{$contract.Type}}.contract.UnpackLog(event, "{{.Original.Name}}", log); err != nil {
 				return nil, err
 			}
+			event.Raw = log
 			return event, nil
 		}
 

accounts/abi/reflect_test.go

@@ -202,12 +202,12 @@ func TestConvertType(t *testing.T) {
 	fields = append(fields, reflect.StructField{
 		Name: "X",
 		Type: reflect.TypeOf(new(big.Int)),
-		Tag:  reflect.StructTag("json:\"" + "x" + "\""),
+		Tag:  "json:\"" + "x" + "\"",
 	})
 	fields = append(fields, reflect.StructField{
 		Name: "Y",
 		Type: reflect.TypeOf(new(big.Int)),
-		Tag:  reflect.StructTag("json:\"" + "y" + "\""),
+		Tag:  "json:\"" + "y" + "\"",
 	})
 	val := reflect.New(reflect.StructOf(fields))
 	val.Elem().Field(0).Set(reflect.ValueOf(big.NewInt(1)))

accounts/abi/type_test.go

@@ -255,7 +255,7 @@ func TestTypeCheck(t *testing.T) {
 		{"bytes", nil, [2]byte{0, 1}, "abi: cannot use array as type slice as argument"},
 		{"bytes", nil, common.Hash{1}, "abi: cannot use array as type slice as argument"},
 		{"string", nil, "hello world", ""},
-		{"string", nil, string(""), ""},
+		{"string", nil, "", ""},
 		{"string", nil, []byte{}, "abi: cannot use slice as type string as argument"},
 		{"bytes32[]", nil, [][32]byte{{}}, ""},
 		{"function", nil, [24]byte{}, ""},

accounts/accounts.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"math/big"
 
-	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/event"

accounts/hd.go

@@ -150,3 +150,31 @@ func (path *DerivationPath) UnmarshalJSON(b []byte) error {
 	*path, err = ParseDerivationPath(dp)
 	return err
 }
+
+// DefaultIterator creates a BIP-32 path iterator, which progresses by increasing the last component:
+// i.e. m/44'/60'/0'/0/0, m/44'/60'/0'/0/1, m/44'/60'/0'/0/2, ... m/44'/60'/0'/0/N.
+func DefaultIterator(base DerivationPath) func() DerivationPath {
+	path := make(DerivationPath, len(base))
+	copy(path[:], base[:])
+	// Set it back by one, so the first call gives the first result
+	path[len(path)-1]--
+	return func() DerivationPath {
+		path[len(path)-1]++
+		return path
+	}
+}
+
+// LedgerLiveIterator creates a bip44 path iterator for Ledger Live.
+// Ledger Live increments the third component rather than the fifth component
+// i.e. m/44'/60'/0'/0/0, m/44'/60'/1'/0/0, m/44'/60'/2'/0/0, ... m/44'/60'/N'/0/0.
+func LedgerLiveIterator(base DerivationPath) func() DerivationPath {
+	path := make(DerivationPath, len(base))
+	copy(path[:], base[:])
+	// Set it back by one, so the first call gives the first result
+	path[2]--
+	return func() DerivationPath {
+		// ledgerLivePathIterator iterates on the third component
+		path[2]++
+		return path
+	}
+}

accounts/hd_test.go

@@ -17,6 +17,7 @@
 package accounts
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 )
@@ -77,3 +78,41 @@ func TestHDPathParsing(t *testing.T) {
 		}
 	}
 }
+
+func testDerive(t *testing.T, next func() DerivationPath, expected []string) {
+	t.Helper()
+	for i, want := range expected {
+		if have := next(); fmt.Sprintf("%v", have) != want {
+			t.Errorf("step %d, have %v, want %v", i, have, want)
+		}
+	}
+}
+
+func TestHdPathIteration(t *testing.T) {
+	testDerive(t, DefaultIterator(DefaultBaseDerivationPath),
+		[]string{
+			"m/44'/60'/0'/0/0", "m/44'/60'/0'/0/1",
+			"m/44'/60'/0'/0/2", "m/44'/60'/0'/0/3",
+			"m/44'/60'/0'/0/4", "m/44'/60'/0'/0/5",
+			"m/44'/60'/0'/0/6", "m/44'/60'/0'/0/7",
+			"m/44'/60'/0'/0/8", "m/44'/60'/0'/0/9",
+		})
+
+	testDerive(t, DefaultIterator(LegacyLedgerBaseDerivationPath),
+		[]string{
+			"m/44'/60'/0'/0", "m/44'/60'/0'/1",
+			"m/44'/60'/0'/2", "m/44'/60'/0'/3",
+			"m/44'/60'/0'/4", "m/44'/60'/0'/5",
+			"m/44'/60'/0'/6", "m/44'/60'/0'/7",
+			"m/44'/60'/0'/8", "m/44'/60'/0'/9",
+		})
+
+	testDerive(t, LedgerLiveIterator(DefaultBaseDerivationPath),
+		[]string{
+			"m/44'/60'/0'/0/0", "m/44'/60'/1'/0/0",
+			"m/44'/60'/2'/0/0", "m/44'/60'/3'/0/0",
+			"m/44'/60'/4'/0/0", "m/44'/60'/5'/0/0",
+			"m/44'/60'/6'/0/0", "m/44'/60'/7'/0/0",
+			"m/44'/60'/8'/0/0", "m/44'/60'/9'/0/0",
+		})
+}

accounts/keystore/account_cache.go

@@ -262,7 +262,7 @@ func (ac *accountCache) scanAccounts() error {
 		switch {
 		case err != nil:
 			log.Debug("Failed to decode keystore key", "path", path, "err", err)
-		case (addr == common.Address{}):
+		case addr == common.Address{}:
 			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
 		default:
 			return &accounts.Account{

accounts/keystore/key.go

@@ -32,7 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 )
 
 const (
@@ -110,7 +110,10 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) {
 	}
 
 	u := new(uuid.UUID)
-	*u = uuid.Parse(keyJSON.Id)
+	*u, err = uuid.Parse(keyJSON.Id)
+	if err != nil {
+		return err
+	}
 	k.Id = *u
 	addr, err := hex.DecodeString(keyJSON.Address)
 	if err != nil {
@@ -128,7 +131,10 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) {
 }
 
 func newKeyFromECDSA(privateKeyECDSA *ecdsa.PrivateKey) *Key {
-	id := uuid.NewRandom()
+	id, err := uuid.NewRandom()
+	if err != nil {
+		panic(fmt.Sprintf("Could not create random uuid: %v", err))
+	}
 	key := &Key{
 		Id:         id,
 		Address:    crypto.PubkeyToAddress(privateKeyECDSA.PublicKey),

accounts/keystore/keystore.go

@@ -283,11 +283,9 @@ func (ks *KeyStore) SignTx(a accounts.Account, tx *types.Transaction, chainID *b
 	if !found {
 		return nil, ErrLocked
 	}
-	// Depending on the presence of the chain ID, sign with EIP155 or homestead
-	if chainID != nil {
-		return types.SignTx(tx, types.NewEIP155Signer(chainID), unlockedKey.PrivateKey)
-	}
-	return types.SignTx(tx, types.HomesteadSigner{}, unlockedKey.PrivateKey)
+	// Depending on the presence of the chain ID, sign with 2718 or homestead
+	signer := types.LatestSignerForChainID(chainID)
+	return types.SignTx(tx, signer, unlockedKey.PrivateKey)
 }
 
 // SignHashWithPassphrase signs hash if the private key matching the given address
@@ -310,12 +308,9 @@ func (ks *KeyStore) SignTxWithPassphrase(a accounts.Account, passphrase string,
 		return nil, err
 	}
 	defer zeroKey(key.PrivateKey)
-
-	// Depending on the presence of the chain ID, sign with EIP155 or homestead
-	if chainID != nil {
-		return types.SignTx(tx, types.NewEIP155Signer(chainID), key.PrivateKey)
-	}
-	return types.SignTx(tx, types.HomesteadSigner{}, key.PrivateKey)
+	// Depending on the presence of the chain ID, sign with or without replay protection.
+	signer := types.LatestSignerForChainID(chainID)
+	return types.SignTx(tx, signer, key.PrivateKey)
 }
 
 // Unlock unlocks the given account indefinitely.

accounts/keystore/passphrase.go

@@ -42,7 +42,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 	"golang.org/x/crypto/pbkdf2"
 	"golang.org/x/crypto/scrypt"
 )
@@ -228,9 +228,12 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 		return nil, err
 	}
 	key := crypto.ToECDSAUnsafe(keyBytes)
-
+	id, err := uuid.FromBytes(keyId)
+	if err != nil {
+		return nil, err
+	}
 	return &Key{
-		Id:         uuid.UUID(keyId),
+		Id:         id,
 		Address:    crypto.PubkeyToAddress(key.PublicKey),
 		PrivateKey: key,
 	}, nil
@@ -276,7 +279,11 @@ func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byt
 	if keyProtected.Version != version {
 		return nil, nil, fmt.Errorf("version not supported: %v", keyProtected.Version)
 	}
-	keyId = uuid.Parse(keyProtected.Id)
+	keyUUID, err := uuid.Parse(keyProtected.Id)
+	if err != nil {
+		return nil, nil, err
+	}
+	keyId = keyUUID[:]
 	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
 	if err != nil {
 		return nil, nil, err
@@ -285,7 +292,11 @@ func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byt
 }
 
 func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byte, keyId []byte, err error) {
-	keyId = uuid.Parse(keyProtected.Id)
+	keyUUID, err := uuid.Parse(keyProtected.Id)
+	if err != nil {
+		return nil, nil, err
+	}
+	keyId = keyUUID[:]
 	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
 	if err != nil {
 		return nil, nil, err

accounts/keystore/presale.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 	"golang.org/x/crypto/pbkdf2"
 )
 
@@ -37,7 +37,10 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou
 	if err != nil {
 		return accounts.Account{}, nil, err
 	}
-	key.Id = uuid.NewRandom()
+	key.Id, err = uuid.NewRandom()
+	if err != nil {
+		return accounts.Account{}, nil, err
+	}
 	a := accounts.Account{
 		Address: key.Address,
 		URL: accounts.URL{
@@ -86,7 +89,7 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 	ecKey := crypto.ToECDSAUnsafe(ethPriv)
 
 	key = &Key{
-		Id:         nil,
+		Id:         uuid.UUID{},
 		Address:    crypto.PubkeyToAddress(ecKey.PublicKey),
 		PrivateKey: ecKey,
 	}

accounts/keystore/wallet.go

@@ -19,7 +19,7 @@ package keystore
 import (
 	"math/big"
 
-	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -58,7 +58,7 @@ func (w *keystoreWallet) Open(passphrase string) error { return nil }
 func (w *keystoreWallet) Close() error { return nil }
 
 // Accounts implements accounts.Wallet, returning an account list consisting of
-// a single account that the plain kestore wallet contains.
+// a single account that the plain keystore wallet contains.
 func (w *keystoreWallet) Accounts() []accounts.Account {
 	return []accounts.Account{w.account}
 }
@@ -93,12 +93,12 @@ func (w *keystoreWallet) signHash(account accounts.Account, hash []byte) ([]byte
 	return w.keystore.SignHash(account, hash)
 }
 
-// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed.
 func (w *keystoreWallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
 	return w.signHash(account, crypto.Keccak256(data))
 }
 
-// SignDataWithPassphrase signs keccak256(data). The mimetype parameter describes the type of data being signed
+// SignDataWithPassphrase signs keccak256(data). The mimetype parameter describes the type of data being signed.
 func (w *keystoreWallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
 	// Make sure the requested account is contained within
 	if !w.Contains(account) {
@@ -108,12 +108,14 @@ func (w *keystoreWallet) SignDataWithPassphrase(account accounts.Account, passph
 	return w.keystore.SignHashWithPassphrase(account, passphrase, crypto.Keccak256(data))
 }
 
+// SignText implements accounts.Wallet, attempting to sign the hash of
+// the given text with the given account.
 func (w *keystoreWallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
 	return w.signHash(account, accounts.TextHash(text))
 }
 
 // SignTextWithPassphrase implements accounts.Wallet, attempting to sign the
-// given hash with the given account using passphrase as extra authentication.
+// hash of the given text with the given account using passphrase as extra authentication.
 func (w *keystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
 	// Make sure the requested account is contained within
 	if !w.Contains(account) {

accounts/scwallet/README.md

@@ -31,12 +31,16 @@
   Write down the URL (`keycard://044def09` in this example). Then ask `geth` to open the wallet:
 
   ```
-  > personal.openWallet("keycard://044def09")
-  Please enter the pairing password:
+  > personal.openWallet("keycard://044def09", "pairing password")
   ```
 
-  Enter the pairing password that you have received during card initialization. Same with the PIN that you will subsequently be
-  asked for.
+  The pairing password has been generated during the card initialization process.
+  
+  The process needs to be repeated once more with the PIN:
+
+  ```
+  > personal.openWallet("keycard://044def09", "PIN number")
+  ```
   
   If everything goes well, you should see your new account when typing `personal` on the console:
 

accounts/scwallet/securechannel.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/sha256"
 	"crypto/sha512"
@@ -27,7 +28,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	pcsc "github.com/gballet/go-libpcsclite"
-	"github.com/wsddn/go-ecdh"
 	"golang.org/x/crypto/pbkdf2"
 	"golang.org/x/text/unicode/norm"
 )
@@ -63,26 +63,19 @@ type SecureChannelSession struct {
 // NewSecureChannelSession creates a new secure channel for the given card and public key.
 func NewSecureChannelSession(card *pcsc.Card, keyData []byte) (*SecureChannelSession, error) {
 	// Generate an ECDSA keypair for ourselves
-	gen := ecdh.NewEllipticECDH(crypto.S256())
-	private, public, err := gen.GenerateKey(rand.Reader)
+	key, err := crypto.GenerateKey()
 	if err != nil {
 		return nil, err
 	}
-
-	cardPublic, ok := gen.Unmarshal(keyData)
-	if !ok {
-		return nil, fmt.Errorf("could not unmarshal public key from card")
-	}
-
-	secret, err := gen.GenerateSharedSecret(private, cardPublic)
+	cardPublic, err := crypto.UnmarshalPubkey(keyData)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not unmarshal public key from card: %v", err)
 	}
-
+	secret, _ := key.Curve.ScalarMult(cardPublic.X, cardPublic.Y, key.D.Bytes())
 	return &SecureChannelSession{
 		card:      card,
-		secret:    secret,
-		publicKey: gen.Marshal(public),
+		secret:    secret.Bytes(),
+		publicKey: elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y),
 	}, nil
 }
 

accounts/scwallet/wallet.go

@@ -33,7 +33,7 @@ import (
 	"sync"
 	"time"
 
-	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -699,7 +699,7 @@ func (w *Wallet) signHash(account accounts.Account, hash []byte) ([]byte, error)
 // the needed details via SignTxWithPassphrase, or by other means (e.g. unlock
 // the account in a keystore).
 func (w *Wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
-	signer := types.NewEIP155Signer(chainID)
+	signer := types.LatestSignerForChainID(chainID)
 	hash := signer.Hash(tx)
 	sig, err := w.signHash(account, hash[:])
 	if err != nil {

accounts/usbwallet/ledger.go

@@ -162,7 +162,7 @@ func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio
 		return common.Address{}, nil, accounts.ErrWalletClosed
 	}
 	// Ensure the wallet is capable of signing the given transaction
-	if chainID != nil && w.version[0] <= 1 && w.version[2] <= 2 {
+	if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 {
 		//lint:ignore ST1005 brand name displayed on the console
 		return common.Address{}, nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
 	}

accounts/usbwallet/trezor.go

@@ -255,9 +255,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 	if chainID == nil {
 		signer = new(types.HomesteadSigner)
 	} else {
+		// Trezor backend does not support typed transactions yet.
 		signer = types.NewEIP155Signer(chainID)
 		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
+
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

accounts/usbwallet/wallet.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"time"
 
-	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"

appveyor.yml

@@ -24,13 +24,13 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.15.windows-%GETH_ARCH%.zip
-  - 7z x go1.15.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://dl.google.com/go/go1.16.windows-%GETH_ARCH%.zip
+  - 7z x go1.16.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 
 build_script:
-  - go run build\ci.go install
+  - go run build\ci.go install -dlgo
 
 after_build:
   - go run build\ci.go archive -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds

build/checksums.txt

@@ -1,6 +1,17 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a  go1.15.src.tar.gz
+7688063d55656105898f323d90a79a39c378d86fe89ae192eb3b7fc46347c95a  go1.16.src.tar.gz
+6000a9522975d116bf76044967d7e69e04e982e9625330d9a539a8b45395f9a8  go1.16.darwin-amd64.tar.gz
+ea435a1ac6d497b03e367fdfb74b33e961d813883468080f6e239b3b03bea6aa  go1.16.linux-386.tar.gz
+013a489ebb3e24ef3d915abe5b94c3286c070dfe0818d5bca8108f1d6e8440d2  go1.16.linux-amd64.tar.gz
+3770f7eb22d05e25fbee8fb53c2a4e897da043eb83c69b9a14f8d98562cd8098  go1.16.linux-arm64.tar.gz
+d1d9404b1dbd77afa2bdc70934e10fbfcf7d785c372efc29462bb7d83d0a32fd  go1.16.linux-armv6l.tar.gz
+481492a17d42193d471b93b7a06da3555331bd833b76336afc87be820c48933f  go1.16.windows-386.zip
+5cc88fa506b3d5c453c54c3ea218fc8dd05d7362ae1de15bb67986b72089ce93  go1.16.windows-amd64.zip
+d7d6c70b05a7c2f68b48aab5ab8cb5116b8444c9ddad131673b152e7cff7c726  go1.16.freebsd-386.tar.gz
+40b03216f6945fb6883a50604fc7f409a83f62171607229a9c598e701e684f8a  go1.16.freebsd-amd64.tar.gz
+27a1aaa988e930b7932ce459c8a63ad5b3333b3a06b016d87ff289f2a11aacd6  go1.16.linux-ppc64le.tar.gz
+be4c9e4e2cf058efc4e3eb013a760cb989ddc4362f111950c990d1c63b27ccbe  go1.16.linux-s390x.tar.gz
 
 d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35  golangci-lint-1.27.0-linux-armv7.tar.gz
 bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4  golangci-lint.exe-1.27.0-windows-amd64.zip

build/ci.go

@@ -26,7 +26,7 @@ Available commands are:
    install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
    test       [ -coverage ] [ packages... ]                                                    -- runs the tests
    lint                                                                                        -- runs certain pre-selected linters
-   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts
+   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts
    importkeys                                                                                  -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
    nsis                                                                                        -- creates a Windows NSIS installer
@@ -46,12 +46,11 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"go/parser"
-	"go/token"
 	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -59,6 +58,7 @@ import (
 	"time"
 
 	"github.com/cespare/cp"
+	"github.com/ethereum/go-ethereum/crypto/signify"
 	"github.com/ethereum/go-ethereum/internal/build"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -135,11 +135,11 @@ var (
 	// Note: artful is unsupported because it was officially deprecated on Launchpad.
 	// Note: cosmic is unsupported because it was officially deprecated on Launchpad.
 	// Note: disco is unsupported because it was officially deprecated on Launchpad.
+	// Note: eoan is unsupported because it was officially deprecated on Launchpad.
 	debDistroGoBoots = map[string]string{
 		"trusty": "golang-1.11",
 		"xenial": "golang-go",
 		"bionic": "golang-go",
-		"eoan":   "golang-go",
 		"focal":  "golang-go",
 		"groovy": "golang-go",
 	}
@@ -148,6 +148,11 @@ var (
 		"golang-1.11": "/usr/lib/go-1.11",
 		"golang-go":   "/usr/lib/go",
 	}
+
+	// This is the version of go that will be downloaded by
+	//
+	//     go run ci.go install -dlgo
+	dlgoVersion = "1.16"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -198,19 +203,19 @@ func main() {
 
 func doInstall(cmdline []string) {
 	var (
+		dlgo = flag.Bool("dlgo", false, "Download Go and build with it")
 		arch = flag.String("arch", "", "Architecture to cross build for")
 		cc   = flag.String("cc", "", "C compiler to cross build with")
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
-	// Check Go version. People regularly open issues about compilation
+	// Check local Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
 	if !strings.Contains(runtime.Version(), "devel") {
 		// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
 		var minor int
 		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
-
 		if minor < 13 {
 			log.Println("You have Go version", runtime.Version())
 			log.Println("go-ethereum requires at least Go version 1.13 and cannot")
@@ -218,90 +223,110 @@ func doInstall(cmdline []string) {
 			os.Exit(1)
 		}
 	}
-	// Compile packages given as arguments, or everything if there are no arguments.
-	packages := []string{"./..."}
-	if flag.NArg() > 0 {
-		packages = flag.Args()
+
+	// Choose which go command we're going to use.
+	var gobuild *exec.Cmd
+	if !*dlgo {
+		// Default behavior: use the go version which runs ci.go right now.
+		gobuild = goTool("build")
+	} else {
+		// Download of Go requested. This is for build environments where the
+		// installed version is too old and cannot be upgraded easily.
+		cachedir := filepath.Join("build", "cache")
+		goroot := downloadGo(runtime.GOARCH, runtime.GOOS, cachedir)
+		gobuild = localGoTool(goroot, "build")
 	}
 
-	if *arch == "" || *arch == runtime.GOARCH {
-		goinstall := goTool("install", buildFlags(env)...)
-		if runtime.GOARCH == "arm64" {
-			goinstall.Args = append(goinstall.Args, "-p", "1")
-		}
-		goinstall.Args = append(goinstall.Args, "-trimpath")
-		goinstall.Args = append(goinstall.Args, "-v")
-		goinstall.Args = append(goinstall.Args, packages...)
-		build.MustRun(goinstall)
-		return
+	// Configure environment for cross build.
+	if *arch != "" || *arch != runtime.GOARCH {
+		gobuild.Env = append(gobuild.Env, "CGO_ENABLED=1")
+		gobuild.Env = append(gobuild.Env, "GOARCH="+*arch)
 	}
 
-	// Seems we are cross compiling, work around forbidden GOBIN
-	goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
-	goinstall.Args = append(goinstall.Args, "-trimpath")
-	goinstall.Args = append(goinstall.Args, "-v")
-	goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
-	goinstall.Args = append(goinstall.Args, packages...)
-	build.MustRun(goinstall)
+	// Configure C compiler.
+	if *cc != "" {
+		gobuild.Env = append(gobuild.Env, "CC="+*cc)
+	} else if os.Getenv("CC") != "" {
+		gobuild.Env = append(gobuild.Env, "CC="+os.Getenv("CC"))
+	}
 
-	if cmds, err := ioutil.ReadDir("cmd"); err == nil {
-		for _, cmd := range cmds {
-			pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
-			if err != nil {
-				log.Fatal(err)
-			}
-			for name := range pkgs {
-				if name == "main" {
-					gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
-					gobuild.Args = append(gobuild.Args, "-v")
-					gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
-					gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
-					build.MustRun(gobuild)
-					break
-				}
-			}
-		}
+	// arm64 CI builders are memory-constrained and can't handle concurrent builds,
+	// better disable it. This check isn't the best, it should probably
+	// check for something in env instead.
+	if runtime.GOARCH == "arm64" {
+		gobuild.Args = append(gobuild.Args, "-p", "1")
+	}
+
+	// Put the default settings in.
+	gobuild.Args = append(gobuild.Args, buildFlags(env)...)
+
+	// We use -trimpath to avoid leaking local paths into the built executables.
+	gobuild.Args = append(gobuild.Args, "-trimpath")
+
+	// Show packages during build.
+	gobuild.Args = append(gobuild.Args, "-v")
+
+	// Now we choose what we're even building.
+	// Default: collect all 'main' packages in cmd/ and build those.
+	packages := flag.Args()
+	if len(packages) == 0 {
+		packages = build.FindMainPackages("./cmd")
+	}
+
+	// Do the build!
+	for _, pkg := range packages {
+		args := make([]string, len(gobuild.Args))
+		copy(args, gobuild.Args)
+		args = append(args, "-o", executablePath(path.Base(pkg)))
+		args = append(args, pkg)
+		build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env})
 	}
 }
 
+// buildFlags returns the go tool flags for building.
 func buildFlags(env build.Environment) (flags []string) {
 	var ld []string
 	if env.Commit != "" {
 		ld = append(ld, "-X", "main.gitCommit="+env.Commit)
 		ld = append(ld, "-X", "main.gitDate="+env.Date)
 	}
+	// Strip DWARF on darwin. This used to be required for certain things,
+	// and there is no downside to this, so we just keep doing it.
 	if runtime.GOOS == "darwin" {
 		ld = append(ld, "-s")
 	}
-
 	if len(ld) > 0 {
 		flags = append(flags, "-ldflags", strings.Join(ld, " "))
 	}
 	return flags
 }
 
+// goTool returns the go tool. This uses the Go version which runs ci.go.
 func goTool(subcmd string, args ...string) *exec.Cmd {
-	return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
+	cmd := build.GoTool(subcmd, args...)
+	goToolSetEnv(cmd)
+	return cmd
 }
 
-func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
-	cmd := build.GoTool(subcmd, args...)
-	if arch == "" || arch == runtime.GOARCH {
-		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
-	} else {
-		cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
-		cmd.Env = append(cmd.Env, "GOARCH="+arch)
-	}
-	if cc != "" {
-		cmd.Env = append(cmd.Env, "CC="+cc)
-	}
+// localGoTool returns the go tool from the given GOROOT.
+func localGoTool(goroot string, subcmd string, args ...string) *exec.Cmd {
+	gotool := filepath.Join(goroot, "bin", "go")
+	cmd := exec.Command(gotool, subcmd)
+	goToolSetEnv(cmd)
+	cmd.Env = append(cmd.Env, "GOROOT="+goroot)
+	cmd.Args = append(cmd.Args, args...)
+	return cmd
+}
+
+// goToolSetEnv forwards the build environment to the go tool.
+func goToolSetEnv(cmd *exec.Cmd) {
+	cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
 	for _, e := range os.Environ() {
-		if strings.HasPrefix(e, "GOBIN=") {
+		if strings.HasPrefix(e, "GOBIN=") || strings.HasPrefix(e, "CC=") {
 			continue
 		}
 		cmd.Env = append(cmd.Env, e)
 	}
-	return cmd
 }
 
 // Running The Tests
@@ -363,7 +388,7 @@ func downloadLinter(cachedir string) string {
 	if err := csdb.DownloadFile(url, archivePath); err != nil {
 		log.Fatal(err)
 	}
-	if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil {
+	if err := build.ExtractArchive(archivePath, cachedir); err != nil {
 		log.Fatal(err)
 	}
 	return filepath.Join(cachedir, base, "golangci-lint")
@@ -372,11 +397,12 @@ func downloadLinter(cachedir string) string {
 // Release Packaging
 func doArchive(cmdline []string) {
 	var (
-		arch   = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
-		atype  = flag.String("type", "zip", "Type of archive to write (zip|tar)")
-		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
-		upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
-		ext    string
+		arch    = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
+		atype   = flag.String("type", "zip", "Type of archive to write (zip|tar)")
+		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
+		signify = flag.String("signify", "", `Environment variable holding the signify key (e.g. LINUX_SIGNIFY_KEY)`)
+		upload  = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
+		ext     string
 	)
 	flag.CommandLine.Parse(cmdline)
 	switch *atype {
@@ -403,7 +429,7 @@ func doArchive(cmdline []string) {
 		log.Fatal(err)
 	}
 	for _, archive := range []string{geth, alltools} {
-		if err := archiveUpload(archive, *upload, *signer); err != nil {
+		if err := archiveUpload(archive, *upload, *signer, *signify); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -423,7 +449,7 @@ func archiveBasename(arch string, archiveVersion string) string {
 	return platform + "-" + archiveVersion
 }
 
-func archiveUpload(archive string, blobstore string, signer string) error {
+func archiveUpload(archive string, blobstore string, signer string, signifyVar string) error {
 	// If signing was requested, generate the signature files
 	if signer != "" {
 		key := getenvBase64(signer)
@@ -431,6 +457,14 @@ func archiveUpload(archive string, blobstore string, signer string) error {
 			return err
 		}
 	}
+	if signifyVar != "" {
+		key := os.Getenv(signifyVar)
+		untrustedComment := "verify with geth-release.pub"
+		trustedComment := fmt.Sprintf("%s (%s)", archive, time.Now().UTC().Format(time.RFC1123))
+		if err := signify.SignFile(archive, archive+".sig", key, untrustedComment, trustedComment); err != nil {
+			return err
+		}
+	}
 	// If uploading to Azure was requested, push the archive possibly with its signature
 	if blobstore != "" {
 		auth := build.AzureBlobstoreConfig{
@@ -446,6 +480,11 @@ func archiveUpload(archive string, blobstore string, signer string) error {
 				return err
 			}
 		}
+		if signifyVar != "" {
+			if err := build.AzureBlobstoreUpload(archive+".sig", filepath.Base(archive+".sig"), auth); err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }
@@ -469,13 +508,12 @@ func maybeSkipArchive(env build.Environment) {
 // Debian Packaging
 func doDebianSource(cmdline []string) {
 	var (
-		goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
-		cachedir  = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`)
-		signer    = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload    = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
-		sshUser   = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
-		workdir   = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
-		now       = time.Now()
+		cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`)
+		signer   = flag.String("signer", "", `Signing key name, also used as package author`)
+		upload   = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
+		sshUser  = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
+		workdir  = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
+		now      = time.Now()
 	)
 	flag.CommandLine.Parse(cmdline)
 	*workdir = makeWorkdir(*workdir)
@@ -490,10 +528,10 @@ func doDebianSource(cmdline []string) {
 	}
 
 	// Download and verify the Go source package.
-	gobundle := downloadGoSources(*goversion, *cachedir)
+	gobundle := downloadGoSources(*cachedir)
 
 	// Download all the dependencies needed to build the sources and run the ci script
-	srcdepfetch := goTool("install", "-n", "./...")
+	srcdepfetch := goTool("mod", "download")
 	srcdepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath"))
 	build.MustRun(srcdepfetch)
 
@@ -509,7 +547,7 @@ func doDebianSource(cmdline []string) {
 			pkgdir := stageDebianSource(*workdir, meta)
 
 			// Add Go source code
-			if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil {
+			if err := build.ExtractArchive(gobundle, pkgdir); err != nil {
 				log.Fatalf("Failed to extract Go sources: %v", err)
 			}
 			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
@@ -541,9 +579,10 @@ func doDebianSource(cmdline []string) {
 	}
 }
 
-func downloadGoSources(version string, cachedir string) string {
+// downloadGoSources downloads the Go source tarball.
+func downloadGoSources(cachedir string) string {
 	csdb := build.MustLoadChecksums("build/checksums.txt")
-	file := fmt.Sprintf("go%s.src.tar.gz", version)
+	file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
 	url := "https://dl.google.com/go/" + file
 	dst := filepath.Join(cachedir, file)
 	if err := csdb.DownloadFile(url, dst); err != nil {
@@ -552,6 +591,41 @@ func downloadGoSources(version string, cachedir string) string {
 	return dst
 }
 
+// downloadGo downloads the Go binary distribution and unpacks it into a temporary
+// directory. It returns the GOROOT of the unpacked toolchain.
+func downloadGo(goarch, goos, cachedir string) string {
+	if goarch == "arm" {
+		goarch = "armv6l"
+	}
+
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	file := fmt.Sprintf("go%s.%s-%s", dlgoVersion, goos, goarch)
+	if goos == "windows" {
+		file += ".zip"
+	} else {
+		file += ".tar.gz"
+	}
+	url := "https://golang.org/dl/" + file
+	dst := filepath.Join(cachedir, file)
+	if err := csdb.DownloadFile(url, dst); err != nil {
+		log.Fatal(err)
+	}
+
+	ucache, err := os.UserCacheDir()
+	if err != nil {
+		log.Fatal(err)
+	}
+	godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", dlgoVersion, goos, goarch))
+	if err := build.ExtractArchive(dst, godir); err != nil {
+		log.Fatal(err)
+	}
+	goroot, err := filepath.Abs(filepath.Join(godir, "go"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	return goroot
+}
+
 func ppaUpload(workdir, ppa, sshUser string, files []string) {
 	p := strings.Split(ppa, "/")
 	if len(p) != 2 {
@@ -747,6 +821,7 @@ func doWindowsInstaller(cmdline []string) {
 	var (
 		arch    = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging")
 		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`)
+		signify = flag.String("signify key", "", `Environment variable holding the signify signing key (e.g. WINDOWS_SIGNIFY_KEY)`)
 		upload  = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
 		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
 	)
@@ -808,7 +883,7 @@ func doWindowsInstaller(cmdline []string) {
 		filepath.Join(*workdir, "geth.nsi"),
 	)
 	// Sign and publish installer.
-	if err := archiveUpload(installer, *upload, *signer); err != nil {
+	if err := archiveUpload(installer, *upload, *signer, *signify); err != nil {
 		log.Fatal(err)
 	}
 }
@@ -817,10 +892,11 @@ func doWindowsInstaller(cmdline []string) {
 
 func doAndroidArchive(cmdline []string) {
 	var (
-		local  = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
-		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`)
-		deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`)
-		upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`)
+		local   = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
+		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`)
+		signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. ANDROID_SIGNIFY_KEY)`)
+		deploy  = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`)
+		upload  = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`)
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
@@ -849,7 +925,7 @@ func doAndroidArchive(cmdline []string) {
 	archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar"
 	os.Rename("geth.aar", archive)
 
-	if err := archiveUpload(archive, *upload, *signer); err != nil {
+	if err := archiveUpload(archive, *upload, *signer, *signify); err != nil {
 		log.Fatal(err)
 	}
 	// Sign and upload all the artifacts to Maven Central
@@ -942,10 +1018,11 @@ func newMavenMetadata(env build.Environment) mavenMetadata {
 
 func doXCodeFramework(cmdline []string) {
 	var (
-		local  = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
-		signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`)
-		deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`)
-		upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
+		local   = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
+		signer  = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`)
+		signify = flag.String("signify", "", `Environment variable holding the signify signing key (e.g. IOS_SIGNIFY_KEY)`)
+		deploy  = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`)
+		upload  = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
@@ -973,14 +1050,14 @@ func doXCodeFramework(cmdline []string) {
 	maybeSkipArchive(env)
 
 	// Sign and upload the framework to Azure
-	if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil {
+	if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil {
 		log.Fatal(err)
 	}
 	// Prepare and upload a PodSpec to CocoaPods
 	if *deploy != "" {
 		meta := newPodMetadata(env, archive)
 		build.Render("build/pod.podspec", "Geth.podspec", 0755, meta)
-		build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose")
+		build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings")
 	}
 }
 

cmd/abigen/main.go

@@ -96,7 +96,7 @@ var (
 	}
 	aliasFlag = cli.StringFlag{
 		Name:  "alias",
-		Usage: "Comma separated aliases for function and event renaming, e.g. foo=bar",
+		Usage: "Comma separated aliases for function and event renaming, e.g. original1=alias1, original2=alias2",
 	}
 )
 

cmd/bootnode/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/discv5"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
@@ -121,17 +120,17 @@ func main() {
 
 	printNotice(&nodeKey.PublicKey, *realaddr)
 
+	db, _ := enode.OpenDB("")
+	ln := enode.NewLocalNode(db, nodeKey)
+	cfg := discover.Config{
+		PrivateKey:  nodeKey,
+		NetRestrict: restrictList,
+	}
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
+		if _, err := discover.ListenV5(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
-		db, _ := enode.OpenDB("")
-		ln := enode.NewLocalNode(db, nodeKey)
-		cfg := discover.Config{
-			PrivateKey:  nodeKey,
-			NetRestrict: restrictList,
-		}
 		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}

cmd/clef/main.go

@@ -54,7 +54,7 @@ import (
 	"github.com/ethereum/go-ethereum/signer/rules"
 	"github.com/ethereum/go-ethereum/signer/storage"
 
-	colorable "github.com/mattn/go-colorable"
+	"github.com/mattn/go-colorable"
 	"github.com/mattn/go-isatty"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -107,11 +107,6 @@ var (
 		Usage: "HTTP-RPC server listening port",
 		Value: node.DefaultHTTPPort + 5,
 	}
-	legacyRPCPortFlag = cli.IntFlag{
-		Name:  "rpcport",
-		Usage: "HTTP-RPC server listening port (Deprecated, please use --http.port).",
-		Value: node.DefaultHTTPPort + 5,
-	}
 	signerSecretFlag = cli.StringFlag{
 		Name:  "signersecret",
 		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
@@ -250,12 +245,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			acceptFlag,
 		},
 	},
-	{
-		Name: "ALIASED (deprecated)",
-		Flags: []cli.Flag{
-			legacyRPCPortFlag,
-		},
-	},
 }
 
 func init() {
@@ -283,7 +272,6 @@ func init() {
 		testFlag,
 		advancedMode,
 		acceptFlag,
-		legacyRPCPortFlag,
 	}
 	app.Action = signer
 	app.Commands = []cli.Command{initCommand,
@@ -677,12 +665,6 @@ func signer(c *cli.Context) error {
 
 		// set port
 		port := c.Int(rpcPortFlag.Name)
-		if c.GlobalIsSet(legacyRPCPortFlag.Name) {
-			if !c.GlobalIsSet(rpcPortFlag.Name) {
-				port = c.Int(legacyRPCPortFlag.Name)
-			}
-			log.Warn("The flag --rpcport is deprecated and will be removed in the future, please use --http.port")
-		}
 
 		// start http server
 		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.HTTPListenAddrFlag.Name), port)
@@ -747,12 +729,10 @@ func DefaultConfigDir() string {
 			appdata := os.Getenv("APPDATA")
 			if appdata != "" {
 				return filepath.Join(appdata, "Signer")
-			} else {
-				return filepath.Join(home, "AppData", "Roaming", "Signer")
 			}
-		} else {
-			return filepath.Join(home, ".clef")
+			return filepath.Join(home, "AppData", "Roaming", "Signer")
 		}
+		return filepath.Join(home, ".clef")
 	}
 	// As we cannot guess a stable location, return empty and handle later
 	return ""
@@ -807,14 +787,16 @@ func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 
 // checkFile is a convenience function to check if a file
 // * exists
-// * is mode 0400
+// * is mode 0400 (unix only)
 func checkFile(filename string) error {
 	info, err := os.Stat(filename)
 	if err != nil {
 		return fmt.Errorf("failed stat on %s: %v", filename, err)
 	}
 	// Check the unix permission bits
-	if info.Mode().Perm()&0377 != 0 {
+	// However, on windows, we cannot use the unix perm-bits, see
+	// https://github.com/ethereum/go-ethereum/issues/20123
+	if runtime.GOOS != "windows" && info.Mode().Perm()&0377 != 0 {
 		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
 	}
 	return nil
@@ -1124,7 +1106,7 @@ func GenDoc(ctx *cli.Context) {
 
 		rlpdata := common.FromHex("0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed")
 		var tx types.Transaction
-		rlp.DecodeBytes(rlpdata, &tx)
+		tx.UnmarshalBinary(rlpdata)
 		add("OnApproved - SignTransactionResult", desc, &ethapi.SignTransactionResult{Raw: rlpdata, Tx: &tx})
 
 	}

cmd/devp2p/README.md

@@ -81,6 +81,37 @@ Now get the ENR of your node and store it in the `NODE` environment variable.
 
 Start the test by running `devp2p discv5 test -listen1 127.0.0.1 -listen2 127.0.0.2 $NODE`.
 
+### Eth Protocol Test Suite
+
+The Eth Protocol test suite is a conformance test suite for the [eth protocol][eth].
+
+To run the eth protocol test suite against your implementation, the node needs to be initialized as such:
+
+1. initialize the geth node with the `genesis.json` file contained in the `testdata` directory
+2. import the `halfchain.rlp` file in the `testdata` directory
+3. run geth with the following flags:
+```
+geth --datadir <datadir> --nodiscover --nat=none --networkid 19763 --verbosity 5
+```
+
+Then, run the following command, replacing `<enode>` with the enode of the geth node: 
+ ```
+ devp2p rlpx eth-test <enode> cmd/devp2p/internal/ethtest/testdata/chain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json
+```
+
+Repeat the above process (re-initialising the node) in order to run the Eth Protocol test suite again.
+
+#### Eth66 Test Suite
+
+The Eth66 test suite is also a conformance test suite for the eth 66 protocol version specifically. 
+To run the eth66 protocol test suite, initialize a geth node as described above and run the following command,
+replacing `<enode>` with the enode of the geth node:
+
+ ```
+ devp2p rlpx eth66-test <enode> cmd/devp2p/internal/ethtest/testdata/chain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json
+```
+
+[eth]: https://github.com/ethereum/devp2p/blob/master/caps/eth.md
 [dns-tutorial]: https://geth.ethereum.org/docs/developers/dns-discovery-setup
 [discv4]: https://github.com/ethereum/devp2p/tree/master/discv4.md
 [discv5]: https://github.com/ethereum/devp2p/tree/master/discv5/discv5.md

cmd/devp2p/discv4cmd.go

@@ -19,14 +19,12 @@ package main
 import (
 	"fmt"
 	"net"
-	"os"
 	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/internal/utesting"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
@@ -82,7 +80,13 @@ var (
 		Name:   "test",
 		Usage:  "Runs tests against a node",
 		Action: discv4Test,
-		Flags:  []cli.Flag{remoteEnodeFlag, testPatternFlag, testListen1Flag, testListen2Flag},
+		Flags: []cli.Flag{
+			remoteEnodeFlag,
+			testPatternFlag,
+			testTAPFlag,
+			testListen1Flag,
+			testListen2Flag,
+		},
 	}
 )
 
@@ -113,20 +117,6 @@ var (
 		Usage:  "Enode of the remote node under test",
 		EnvVar: "REMOTE_ENODE",
 	}
-	testPatternFlag = cli.StringFlag{
-		Name:  "run",
-		Usage: "Pattern of test suite(s) to run",
-	}
-	testListen1Flag = cli.StringFlag{
-		Name:  "listen1",
-		Usage: "IP address of the first tester",
-		Value: v4test.Listen1,
-	}
-	testListen2Flag = cli.StringFlag{
-		Name:  "listen2",
-		Usage: "IP address of the second tester",
-		Value: v4test.Listen2,
-	}
 )
 
 func discv4Ping(ctx *cli.Context) error {
@@ -213,6 +203,7 @@ func discv4Crawl(ctx *cli.Context) error {
 	return nil
 }
 
+// discv4Test runs the protocol test suite.
 func discv4Test(ctx *cli.Context) error {
 	// Configure test package globals.
 	if !ctx.IsSet(remoteEnodeFlag.Name) {
@@ -221,18 +212,7 @@ func discv4Test(ctx *cli.Context) error {
 	v4test.Remote = ctx.String(remoteEnodeFlag.Name)
 	v4test.Listen1 = ctx.String(testListen1Flag.Name)
 	v4test.Listen2 = ctx.String(testListen2Flag.Name)
-
-	// Filter and run test cases.
-	tests := v4test.AllTests
-	if ctx.IsSet(testPatternFlag.Name) {
-		tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
-	}
-	results := utesting.RunTests(tests, os.Stdout)
-	if fails := utesting.CountFailures(results); fails > 0 {
-		return fmt.Errorf("%v/%v tests passed.", len(tests)-fails, len(tests))
-	}
-	fmt.Printf("%v/%v passed\n", len(tests), len(tests))
-	return nil
+	return runTests(ctx, v4test.AllTests)
 }
 
 // startV4 starts an ephemeral discovery V4 node.

cmd/devp2p/discv5cmd.go

@@ -18,13 +18,10 @@ package main
 
 import (
 	"fmt"
-	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/internal/utesting"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -62,7 +59,12 @@ var (
 		Name:   "test",
 		Usage:  "Runs protocol tests against a node",
 		Action: discv5Test,
-		Flags:  []cli.Flag{testPatternFlag, testListen1Flag, testListen2Flag},
+		Flags: []cli.Flag{
+			testPatternFlag,
+			testTAPFlag,
+			testListen1Flag,
+			testListen2Flag,
+		},
 	}
 	discv5ListenCommand = cli.Command{
 		Name:   "listen",
@@ -114,28 +116,14 @@ func discv5Crawl(ctx *cli.Context) error {
 	return nil
 }
 
+// discv5Test runs the protocol test suite.
 func discv5Test(ctx *cli.Context) error {
-	// Disable logging unless explicitly enabled.
-	if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") {
-		log.Root().SetHandler(log.DiscardHandler())
-	}
-
-	// Filter and run test cases.
 	suite := &v5test.Suite{
 		Dest:    getNodeArg(ctx),
 		Listen1: ctx.String(testListen1Flag.Name),
 		Listen2: ctx.String(testListen2Flag.Name),
 	}
-	tests := suite.AllTests()
-	if ctx.IsSet(testPatternFlag.Name) {
-		tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
-	}
-	results := utesting.RunTests(tests, os.Stdout)
-	if fails := utesting.CountFailures(results); fails > 0 {
-		return fmt.Errorf("%v/%v tests passed.", len(tests)-fails, len(tests))
-	}
-	fmt.Printf("%v/%v passed\n", len(tests), len(tests))
-	return nil
+	return runTests(ctx, suite.AllTests())
 }
 
 func discv5Listen(ctx *cli.Context) error {

cmd/devp2p/dnscmd.go

@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/p2p/dnsdisc"
 	"github.com/ethereum/go-ethereum/p2p/enode"
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 )
 
 var (

cmd/devp2p/internal/ethtest/chain.go

@@ -124,32 +124,6 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) {
 // loadChain takes the given chain.rlp file, and decodes and returns
 // the blocks from the file.
 func loadChain(chainfile string, genesis string) (*Chain, error) {
-	// Open the file handle and potentially unwrap the gzip stream
-	fh, err := os.Open(chainfile)
-	if err != nil {
-		return nil, err
-	}
-	defer fh.Close()
-
-	var reader io.Reader = fh
-	if strings.HasSuffix(chainfile, ".gz") {
-		if reader, err = gzip.NewReader(reader); err != nil {
-			return nil, err
-		}
-	}
-	stream := rlp.NewStream(reader, 0)
-	var blocks []*types.Block
-	for i := 0; ; i++ {
-		var b types.Block
-		if err := stream.Decode(&b); err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, fmt.Errorf("at block %d: %v", i, err)
-		}
-		blocks = append(blocks, &b)
-	}
-
-	// Open the file handle and potentially unwrap the gzip stream
 	chainConfig, err := ioutil.ReadFile(genesis)
 	if err != nil {
 		return nil, err
@@ -158,9 +132,36 @@ func loadChain(chainfile string, genesis string) (*Chain, error) {
 	if err := json.Unmarshal(chainConfig, &gen); err != nil {
 		return nil, err
 	}
+	gblock := gen.ToBlock(nil)
 
-	return &Chain{
-		blocks:      blocks,
-		chainConfig: gen.Config,
-	}, nil
+	// Load chain.rlp.
+	fh, err := os.Open(chainfile)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	var reader io.Reader = fh
+	if strings.HasSuffix(chainfile, ".gz") {
+		if reader, err = gzip.NewReader(reader); err != nil {
+			return nil, err
+		}
+	}
+	stream := rlp.NewStream(reader, 0)
+	var blocks = make([]*types.Block, 1)
+	blocks[0] = gblock
+	for i := 0; ; i++ {
+		var b types.Block
+		if err := stream.Decode(&b); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, fmt.Errorf("at block index %d: %v", i, err)
+		}
+		if b.NumberU64() != uint64(i+1) {
+			return nil, fmt.Errorf("block at index %d has wrong number %d", i, b.NumberU64())
+		}
+		blocks = append(blocks, &b)
+	}
+
+	c := &Chain{blocks: blocks, chainConfig: gen.Config}
+	return c, nil
 }

cmd/devp2p/internal/ethtest/chain_test.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/stretchr/testify/assert"
 )
@@ -73,7 +74,7 @@ func TestEthProtocolNegotiation(t *testing.T) {
 // TestChain_GetHeaders tests whether the test suite can correctly
 // respond to a GetBlockHeaders request from a node.
 func TestChain_GetHeaders(t *testing.T) {
-	chainFile, err := filepath.Abs("./testdata/chain.rlp.gz")
+	chainFile, err := filepath.Abs("./testdata/chain.rlp")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,7 +94,7 @@ func TestChain_GetHeaders(t *testing.T) {
 	}{
 		{
 			req: GetBlockHeaders{
-				Origin: hashOrNumber{
+				Origin: eth.HashOrNumber{
 					Number: uint64(2),
 				},
 				Amount:  uint64(5),
@@ -110,7 +111,7 @@ func TestChain_GetHeaders(t *testing.T) {
 		},
 		{
 			req: GetBlockHeaders{
-				Origin: hashOrNumber{
+				Origin: eth.HashOrNumber{
 					Number: uint64(chain.Len() - 1),
 				},
 				Amount:  uint64(3),
@@ -125,7 +126,7 @@ func TestChain_GetHeaders(t *testing.T) {
 		},
 		{
 			req: GetBlockHeaders{
-				Origin: hashOrNumber{
+				Origin: eth.HashOrNumber{
 					Hash: chain.Head().Hash(),
 				},
 				Amount:  uint64(1),

cmd/devp2p/internal/ethtest/eth66_suite.go

@@ -0,0 +1,382 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethtest
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/p2p"
+)
+
+// TestStatus_66 attempts to connect to the given node and exchange
+// a status message with it on the eth66 protocol, and then check to
+// make sure the chain head is correct.
+func (s *Suite) TestStatus_66(t *utesting.T) {
+	conn := s.dial66(t)
+	// get protoHandshake
+	conn.handshake(t)
+	// get status
+	switch msg := conn.statusExchange66(t, s.chain).(type) {
+	case *Status:
+		status := *msg
+		if status.ProtocolVersion != uint32(66) {
+			t.Fatalf("mismatch in version: wanted 66, got %d", status.ProtocolVersion)
+		}
+		t.Logf("got status message: %s", pretty.Sdump(msg))
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+	}
+}
+
+// TestGetBlockHeaders_66 tests whether the given node can respond to
+// an eth66 `GetBlockHeaders` request and that the response is accurate.
+func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) {
+	conn := s.setupConnection66(t)
+	// get block headers
+	req := &eth.GetBlockHeadersPacket66{
+		RequestId: 3,
+		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+			Origin: eth.HashOrNumber{
+				Hash: s.chain.blocks[1].Hash(),
+			},
+			Amount:  2,
+			Skip:    1,
+			Reverse: false,
+		},
+	}
+	// write message
+	headers := s.getBlockHeaders66(t, conn, req, req.RequestId)
+	// check for correct headers
+	headersMatch(t, s.chain, headers)
+}
+
+// TestSimultaneousRequests_66 sends two simultaneous `GetBlockHeader` requests
+// with different request IDs and checks to make sure the node responds with the correct
+// headers per request.
+func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) {
+	// create two connections
+	conn1, conn2 := s.setupConnection66(t), s.setupConnection66(t)
+	// create two requests
+	req1 := &eth.GetBlockHeadersPacket66{
+		RequestId: 111,
+		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+			Origin: eth.HashOrNumber{
+				Hash: s.chain.blocks[1].Hash(),
+			},
+			Amount:  2,
+			Skip:    1,
+			Reverse: false,
+		},
+	}
+	req2 := &eth.GetBlockHeadersPacket66{
+		RequestId: 222,
+		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+			Origin: eth.HashOrNumber{
+				Hash: s.chain.blocks[1].Hash(),
+			},
+			Amount:  4,
+			Skip:    1,
+			Reverse: false,
+		},
+	}
+	// wait for headers for first request
+	headerChan := make(chan BlockHeaders, 1)
+	go func(headers chan BlockHeaders) {
+		headers <- s.getBlockHeaders66(t, conn1, req1, req1.RequestId)
+	}(headerChan)
+	// check headers of second request
+	headersMatch(t, s.chain, s.getBlockHeaders66(t, conn2, req2, req2.RequestId))
+	// check headers of first request
+	headersMatch(t, s.chain, <-headerChan)
+}
+
+// TestBroadcast_66 tests whether a block announcement is correctly
+// propagated to the given node's peer(s) on the eth66 protocol.
+func (s *Suite) TestBroadcast_66(t *utesting.T) {
+	sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t)
+	nextBlock := len(s.chain.blocks)
+	blockAnnouncement := &NewBlock{
+		Block: s.fullChain.blocks[nextBlock],
+		TD:    s.fullChain.TD(nextBlock + 1),
+	}
+	s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement)
+	// update test suite chain
+	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
+	// wait for client to update its chain
+	if err := receiveConn.waitForBlock66(s.chain.Head()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestGetBlockBodies_66 tests whether the given node can respond to
+// a `GetBlockBodies` request and that the response is accurate over
+// the eth66 protocol.
+func (s *Suite) TestGetBlockBodies_66(t *utesting.T) {
+	conn := s.setupConnection66(t)
+	// create block bodies request
+	id := uint64(55)
+	req := &eth.GetBlockBodiesPacket66{
+		RequestId: id,
+		GetBlockBodiesPacket: eth.GetBlockBodiesPacket{
+			s.chain.blocks[54].Hash(),
+			s.chain.blocks[75].Hash(),
+		},
+	}
+	if err := conn.write66(req, GetBlockBodies{}.Code()); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+
+	reqID, msg := conn.readAndServe66(s.chain, timeout)
+	switch msg := msg.(type) {
+	case BlockBodies:
+		if reqID != req.RequestId {
+			t.Fatalf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID)
+		}
+		t.Logf("received %d block bodies", len(msg))
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+	}
+}
+
+// TestLargeAnnounce_66 tests the announcement mechanism with a large block.
+func (s *Suite) TestLargeAnnounce_66(t *utesting.T) {
+	nextBlock := len(s.chain.blocks)
+	blocks := []*NewBlock{
+		{
+			Block: largeBlock(),
+			TD:    s.fullChain.TD(nextBlock + 1),
+		},
+		{
+			Block: s.fullChain.blocks[nextBlock],
+			TD:    largeNumber(2),
+		},
+		{
+			Block: largeBlock(),
+			TD:    largeNumber(2),
+		},
+		{
+			Block: s.fullChain.blocks[nextBlock],
+			TD:    s.fullChain.TD(nextBlock + 1),
+		},
+	}
+
+	for i, blockAnnouncement := range blocks[0:3] {
+		t.Logf("Testing malicious announcement: %v\n", i)
+		sendConn := s.setupConnection66(t)
+		if err := sendConn.Write(blockAnnouncement); err != nil {
+			t.Fatalf("could not write to connection: %v", err)
+		}
+		// Invalid announcement, check that peer disconnected
+		switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) {
+		case *Disconnect:
+		case *Error:
+			break
+		default:
+			t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg))
+		}
+	}
+	// Test the last block as a valid block
+	sendConn := s.setupConnection66(t)
+	receiveConn := s.setupConnection66(t)
+	s.testAnnounce66(t, sendConn, receiveConn, blocks[3])
+	// update test suite chain
+	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
+	// wait for client to update its chain
+	if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestMaliciousHandshake_66 tries to send malicious data during the handshake.
+func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) {
+	conn := s.dial66(t)
+	// write hello to client
+	pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:]
+	handshakes := []*Hello{
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: largeString(2), Version: 66},
+			},
+			ID: pub0,
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+				{Name: "eth", Version: 66},
+			},
+			ID: append(pub0, byte(0)),
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+				{Name: "eth", Version: 66},
+			},
+			ID: append(pub0, pub0...),
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+				{Name: "eth", Version: 66},
+			},
+			ID: largeBuffer(2),
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: largeString(2), Version: 66},
+			},
+			ID: largeBuffer(2),
+		},
+	}
+	for i, handshake := range handshakes {
+		t.Logf("Testing malicious handshake %v\n", i)
+		// Init the handshake
+		if err := conn.Write(handshake); err != nil {
+			t.Fatalf("could not write to connection: %v", err)
+		}
+		// check that the peer disconnected
+		timeout := 20 * time.Second
+		// Discard one hello
+		for i := 0; i < 2; i++ {
+			switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
+			case *Disconnect:
+			case *Error:
+			case *Hello:
+				// Hello's are sent concurrently, so ignore them
+				continue
+			default:
+				t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+			}
+		}
+		// Dial for the next round
+		conn = s.dial66(t)
+	}
+}
+
+// TestMaliciousStatus_66 sends a status package with a large total difficulty.
+func (s *Suite) TestMaliciousStatus_66(t *utesting.T) {
+	conn := s.dial66(t)
+	// get protoHandshake
+	conn.handshake(t)
+	status := &Status{
+		ProtocolVersion: uint32(66),
+		NetworkID:       s.chain.chainConfig.ChainID.Uint64(),
+		TD:              largeNumber(2),
+		Head:            s.chain.blocks[s.chain.Len()-1].Hash(),
+		Genesis:         s.chain.blocks[0].Hash(),
+		ForkID:          s.chain.ForkID(),
+	}
+	// get status
+	switch msg := conn.statusExchange(t, s.chain, status).(type) {
+	case *Status:
+		t.Logf("%+v\n", msg)
+	default:
+		t.Fatalf("expected status, got: %#v ", msg)
+	}
+	// wait for disconnect
+	switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
+	case *Disconnect:
+	case *Error:
+		return
+	default:
+		t.Fatalf("expected disconnect, got: %s", pretty.Sdump(msg))
+	}
+}
+
+func (s *Suite) TestTransaction_66(t *utesting.T) {
+	tests := []*types.Transaction{
+		getNextTxFromChain(t, s),
+		unknownTx(t, s),
+	}
+	for i, tx := range tests {
+		t.Logf("Testing tx propagation: %v\n", i)
+		sendSuccessfulTx66(t, s, tx)
+	}
+}
+
+func (s *Suite) TestMaliciousTx_66(t *utesting.T) {
+	tests := []*types.Transaction{
+		getOldTxFromChain(t, s),
+		invalidNonceTx(t, s),
+		hugeAmount(t, s),
+		hugeGasPrice(t, s),
+		hugeData(t, s),
+	}
+	for i, tx := range tests {
+		t.Logf("Testing malicious tx propagation: %v\n", i)
+		sendFailingTx66(t, s, tx)
+	}
+}
+
+// TestZeroRequestID_66 checks that a request ID of zero is still handled
+// by the node.
+func (s *Suite) TestZeroRequestID_66(t *utesting.T) {
+	conn := s.setupConnection66(t)
+	req := &eth.GetBlockHeadersPacket66{
+		RequestId: 0,
+		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+			Origin: eth.HashOrNumber{
+				Number: 0,
+			},
+			Amount: 2,
+		},
+	}
+	headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req, req.RequestId))
+}
+
+// TestSameRequestID_66 sends two requests with the same request ID
+// concurrently to a single node.
+func (s *Suite) TestSameRequestID_66(t *utesting.T) {
+	conn := s.setupConnection66(t)
+	// create two separate requests with same ID
+	reqID := uint64(1234)
+	req1 := &eth.GetBlockHeadersPacket66{
+		RequestId: reqID,
+		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+			Origin: eth.HashOrNumber{
+				Number: 0,
+			},
+			Amount: 2,
+		},
+	}
+	req2 := &eth.GetBlockHeadersPacket66{
+		RequestId: reqID,
+		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+			Origin: eth.HashOrNumber{
+				Number: 33,
+			},
+			Amount: 2,
+		},
+	}
+	// send requests concurrently
+	go func() {
+		headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req2, reqID))
+	}()
+	// check response from first request
+	headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req1, reqID))
+}

cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go

@@ -0,0 +1,270 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethtest
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/stretchr/testify/assert"
+)
+
+func (c *Conn) statusExchange66(t *utesting.T, chain *Chain) Message {
+	status := &Status{
+		ProtocolVersion: uint32(66),
+		NetworkID:       chain.chainConfig.ChainID.Uint64(),
+		TD:              chain.TD(chain.Len()),
+		Head:            chain.blocks[chain.Len()-1].Hash(),
+		Genesis:         chain.blocks[0].Hash(),
+		ForkID:          chain.ForkID(),
+	}
+	return c.statusExchange(t, chain, status)
+}
+
+func (s *Suite) dial66(t *utesting.T) *Conn {
+	conn, err := s.dial()
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66})
+	return conn
+}
+
+func (c *Conn) write66(req eth.Packet, code int) error {
+	payload, err := rlp.EncodeToBytes(req)
+	if err != nil {
+		return err
+	}
+	_, err = c.Conn.Write(uint64(code), payload)
+	return err
+}
+
+func (c *Conn) read66() (uint64, Message) {
+	code, rawData, _, err := c.Conn.Read()
+	if err != nil {
+		return 0, errorf("could not read from connection: %v", err)
+	}
+
+	var msg Message
+
+	switch int(code) {
+	case (Hello{}).Code():
+		msg = new(Hello)
+
+	case (Ping{}).Code():
+		msg = new(Ping)
+	case (Pong{}).Code():
+		msg = new(Pong)
+	case (Disconnect{}).Code():
+		msg = new(Disconnect)
+	case (Status{}).Code():
+		msg = new(Status)
+	case (GetBlockHeaders{}).Code():
+		ethMsg := new(eth.GetBlockHeadersPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket)
+	case (BlockHeaders{}).Code():
+		ethMsg := new(eth.BlockHeadersPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket)
+	case (GetBlockBodies{}).Code():
+		ethMsg := new(eth.GetBlockBodiesPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket)
+	case (BlockBodies{}).Code():
+		ethMsg := new(eth.BlockBodiesPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket)
+	case (NewBlock{}).Code():
+		msg = new(NewBlock)
+	case (NewBlockHashes{}).Code():
+		msg = new(NewBlockHashes)
+	case (Transactions{}).Code():
+		msg = new(Transactions)
+	case (NewPooledTransactionHashes{}).Code():
+		msg = new(NewPooledTransactionHashes)
+	default:
+		msg = errorf("invalid message code: %d", code)
+	}
+
+	if msg != nil {
+		if err := rlp.DecodeBytes(rawData, msg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return 0, msg
+	}
+	return 0, errorf("invalid message: %s", string(rawData))
+}
+
+// ReadAndServe serves GetBlockHeaders requests while waiting
+// on another message from the node.
+func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) {
+	start := time.Now()
+	for time.Since(start) < timeout {
+		timeout := time.Now().Add(10 * time.Second)
+		c.SetReadDeadline(timeout)
+
+		reqID, msg := c.read66()
+
+		switch msg := msg.(type) {
+		case *Ping:
+			c.Write(&Pong{})
+		case *GetBlockHeaders:
+			headers, err := chain.GetHeaders(*msg)
+			if err != nil {
+				return 0, errorf("could not get headers for inbound header request: %v", err)
+			}
+
+			if err := c.Write(headers); err != nil {
+				return 0, errorf("could not write to connection: %v", err)
+			}
+		default:
+			return reqID, msg
+		}
+	}
+	return 0, errorf("no message received within %v", timeout)
+}
+
+func (s *Suite) setupConnection66(t *utesting.T) *Conn {
+	// create conn
+	sendConn := s.dial66(t)
+	sendConn.handshake(t)
+	sendConn.statusExchange66(t, s.chain)
+	return sendConn
+}
+
+func (s *Suite) testAnnounce66(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) {
+	// Announce the block.
+	if err := sendConn.Write(blockAnnouncement); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	s.waitAnnounce66(t, receiveConn, blockAnnouncement)
+}
+
+func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) {
+	timeout := 20 * time.Second
+	_, msg := conn.readAndServe66(s.chain, timeout)
+	switch msg := msg.(type) {
+	case *NewBlock:
+		t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block))
+		assert.Equal(t,
+			blockAnnouncement.Block.Header(), msg.Block.Header(),
+			"wrong block header in announcement",
+		)
+		assert.Equal(t,
+			blockAnnouncement.TD, msg.TD,
+			"wrong TD in announcement",
+		)
+	case *NewBlockHashes:
+		blockHashes := *msg
+		t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes))
+		assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash,
+			"wrong block hash in announcement",
+		)
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+	}
+}
+
+// waitForBlock66 waits for confirmation from the client that it has
+// imported the given block.
+func (c *Conn) waitForBlock66(block *types.Block) error {
+	defer c.SetReadDeadline(time.Time{})
+
+	timeout := time.Now().Add(20 * time.Second)
+	c.SetReadDeadline(timeout)
+	for {
+		req := eth.GetBlockHeadersPacket66{
+			RequestId: 54,
+			GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+				Origin: eth.HashOrNumber{
+					Hash: block.Hash(),
+				},
+				Amount: 1,
+			},
+		}
+		if err := c.write66(req, GetBlockHeaders{}.Code()); err != nil {
+			return err
+		}
+
+		reqID, msg := c.read66()
+		// check message
+		switch msg := msg.(type) {
+		case BlockHeaders:
+			// check request ID
+			if reqID != req.RequestId {
+				return fmt.Errorf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID)
+			}
+			if len(msg) > 0 {
+				return nil
+			}
+			time.Sleep(100 * time.Millisecond)
+		default:
+			return fmt.Errorf("invalid message: %s", pretty.Sdump(msg))
+		}
+	}
+}
+
+func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) {
+	sendConn := s.setupConnection66(t)
+	sendSuccessfulTxWithConn(t, s, tx, sendConn)
+}
+
+func sendFailingTx66(t *utesting.T, s *Suite, tx *types.Transaction) {
+	sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t)
+	sendFailingTxWithConns(t, s, tx, sendConn, recvConn)
+}
+
+func (s *Suite) getBlockHeaders66(t *utesting.T, conn *Conn, req eth.Packet, expectedID uint64) BlockHeaders {
+	if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	// check block headers response
+	reqID, msg := conn.readAndServe66(s.chain, timeout)
+
+	switch msg := msg.(type) {
+	case BlockHeaders:
+		if reqID != expectedID {
+			t.Fatalf("request ID mismatch: wanted %d, got %d", expectedID, reqID)
+		}
+		return msg
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+		return nil
+	}
+}
+
+func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) {
+	for _, header := range headers {
+		num := header.Number.Uint64()
+		t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash()))
+		assert.Equal(t, chain.blocks[int(num)].Header(), header)
+	}
+}

cmd/devp2p/internal/ethtest/large.go

@@ -0,0 +1,80 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethtest
+
+import (
+	"crypto/rand"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// largeNumber returns a very large big.Int.
+func largeNumber(megabytes int) *big.Int {
+	buf := make([]byte, megabytes*1024*1024)
+	rand.Read(buf)
+	bigint := new(big.Int)
+	bigint.SetBytes(buf)
+	return bigint
+}
+
+// largeBuffer returns a very large buffer.
+func largeBuffer(megabytes int) []byte {
+	buf := make([]byte, megabytes*1024*1024)
+	rand.Read(buf)
+	return buf
+}
+
+// largeString returns a very large string.
+func largeString(megabytes int) string {
+	buf := make([]byte, megabytes*1024*1024)
+	rand.Read(buf)
+	return hexutil.Encode(buf)
+}
+
+func largeBlock() *types.Block {
+	return types.NewBlockWithHeader(largeHeader())
+}
+
+// Returns a random hash
+func randHash() common.Hash {
+	var h common.Hash
+	rand.Read(h[:])
+	return h
+}
+
+func largeHeader() *types.Header {
+	return &types.Header{
+		MixDigest:   randHash(),
+		ReceiptHash: randHash(),
+		TxHash:      randHash(),
+		Nonce:       types.BlockNonce{},
+		Extra:       []byte{},
+		Bloom:       types.Bloom{},
+		GasUsed:     0,
+		Coinbase:    common.Address{},
+		GasLimit:    0,
+		UncleHash:   randHash(),
+		Time:        1337,
+		ParentHash:  randHash(),
+		Root:        randHash(),
+		Number:      largeNumber(2),
+		Difficulty:  largeNumber(2),
+	}
+}

cmd/devp2p/internal/ethtest/suite.go

@@ -19,14 +19,28 @@ package ethtest
 import (
 	"fmt"
 	"net"
+	"time"
 
+	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/rlpx"
 	"github.com/stretchr/testify/assert"
 )
 
+var pretty = spew.ConfigState{
+	Indent:                  "  ",
+	DisableCapacities:       true,
+	DisablePointerAddresses: true,
+	SortKeys:                true,
+}
+
+var timeout = 20 * time.Second
+
 // Suite represents a structure used to test the eth
 // protocol of a node(s).
 type Suite struct {
@@ -39,24 +53,47 @@ type Suite struct {
 // NewSuite creates and returns a new eth-test suite that can
 // be used to test the given node against the given blockchain
 // data.
-func NewSuite(dest *enode.Node, chainfile string, genesisfile string) *Suite {
+func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, error) {
 	chain, err := loadChain(chainfile, genesisfile)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 	return &Suite{
 		Dest:      dest,
 		chain:     chain.Shorten(1000),
 		fullChain: chain,
-	}
+	}, nil
 }
 
-func (s *Suite) AllTests() []utesting.Test {
+func (s *Suite) EthTests() []utesting.Test {
 	return []utesting.Test{
+		// status
 		{Name: "Status", Fn: s.TestStatus},
+		{Name: "Status_66", Fn: s.TestStatus_66},
+		// get block headers
 		{Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders},
-		{Name: "Broadcast", Fn: s.TestBroadcast},
+		{Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66},
+		{Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66},
+		{Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66},
+		{Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66},
+		// get block bodies
 		{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
+		{Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66},
+		// broadcast
+		{Name: "Broadcast", Fn: s.TestBroadcast},
+		{Name: "Broadcast_66", Fn: s.TestBroadcast_66},
+		{Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce},
+		{Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66},
+		// malicious handshakes + status
+		{Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake},
+		{Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus},
+		{Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66},
+		{Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus},
+		// test transactions
+		{Name: "TestTransactions", Fn: s.TestTransaction},
+		{Name: "TestTransactions_66", Fn: s.TestTransaction_66},
+		{Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx},
+		{Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66},
 	}
 }
 
@@ -71,11 +108,44 @@ func (s *Suite) TestStatus(t *utesting.T) {
 	// get protoHandshake
 	conn.handshake(t)
 	// get status
-	switch msg := conn.statusExchange(t, s.chain).(type) {
+	switch msg := conn.statusExchange(t, s.chain, nil).(type) {
+	case *Status:
+		t.Logf("got status message: %s", pretty.Sdump(msg))
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+	}
+}
+
+// TestMaliciousStatus sends a status package with a large total difficulty.
+func (s *Suite) TestMaliciousStatus(t *utesting.T) {
+	conn, err := s.dial()
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	// get protoHandshake
+	conn.handshake(t)
+	status := &Status{
+		ProtocolVersion: uint32(conn.ethProtocolVersion),
+		NetworkID:       s.chain.chainConfig.ChainID.Uint64(),
+		TD:              largeNumber(2),
+		Head:            s.chain.blocks[s.chain.Len()-1].Hash(),
+		Genesis:         s.chain.blocks[0].Hash(),
+		ForkID:          s.chain.ForkID(),
+	}
+	// get status
+	switch msg := conn.statusExchange(t, s.chain, status).(type) {
 	case *Status:
 		t.Logf("%+v\n", msg)
 	default:
-		t.Fatalf("unexpected: %#v", msg)
+		t.Fatalf("expected status, got: %#v ", msg)
+	}
+	// wait for disconnect
+	switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
+	case *Disconnect:
+	case *Error:
+		return
+	default:
+		t.Fatalf("expected disconnect, got: %s", pretty.Sdump(msg))
 	}
 }
 
@@ -88,11 +158,11 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 	}
 
 	conn.handshake(t)
-	conn.statusExchange(t, s.chain)
+	conn.statusExchange(t, s.chain, nil)
 
 	// get block headers
 	req := &GetBlockHeaders{
-		Origin: hashOrNumber{
+		Origin: eth.HashOrNumber{
 			Hash: s.chain.blocks[1].Hash(),
 		},
 		Amount:  2,
@@ -104,16 +174,16 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 		t.Fatalf("could not write to connection: %v", err)
 	}
 
-	switch msg := conn.ReadAndServe(s.chain).(type) {
+	switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
 	case *BlockHeaders:
-		headers := msg
-		for _, header := range *headers {
+		headers := *msg
+		for _, header := range headers {
 			num := header.Number.Uint64()
+			t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash()))
 			assert.Equal(t, s.chain.blocks[int(num)].Header(), header)
-			t.Logf("\nHEADER FOR BLOCK NUMBER %d: %+v\n", header.Number, header)
 		}
 	default:
-		t.Fatalf("unexpected: %#v", msg)
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
 	}
 }
 
@@ -126,91 +196,255 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 	}
 
 	conn.handshake(t)
-	conn.statusExchange(t, s.chain)
+	conn.statusExchange(t, s.chain, nil)
 	// create block bodies request
-	req := &GetBlockBodies{s.chain.blocks[54].Hash(), s.chain.blocks[75].Hash()}
+	req := &GetBlockBodies{
+		s.chain.blocks[54].Hash(),
+		s.chain.blocks[75].Hash(),
+	}
 	if err := conn.Write(req); err != nil {
 		t.Fatalf("could not write to connection: %v", err)
 	}
 
-	switch msg := conn.ReadAndServe(s.chain).(type) {
+	switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
 	case *BlockBodies:
-		bodies := msg
-		for _, body := range *bodies {
-			t.Logf("\nBODY: %+v\n", body)
-		}
+		t.Logf("received %d block bodies", len(*msg))
 	default:
-		t.Fatalf("unexpected: %#v", msg)
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
 	}
 }
 
 // TestBroadcast tests whether a block announcement is correctly
 // propagated to the given node's peer(s).
 func (s *Suite) TestBroadcast(t *utesting.T) {
-	// create conn to send block announcement
-	sendConn, err := s.dial()
-	if err != nil {
-		t.Fatalf("could not dial: %v", err)
-	}
-	// create conn to receive block announcement
-	receiveConn, err := s.dial()
-	if err != nil {
-		t.Fatalf("could not dial: %v", err)
-	}
-
-	sendConn.handshake(t)
-	receiveConn.handshake(t)
-
-	sendConn.statusExchange(t, s.chain)
-	receiveConn.statusExchange(t, s.chain)
-
-	// sendConn sends the block announcement
+	sendConn, receiveConn := s.setupConnection(t), s.setupConnection(t)
+	nextBlock := len(s.chain.blocks)
 	blockAnnouncement := &NewBlock{
-		Block: s.fullChain.blocks[1000],
-		TD:    s.fullChain.TD(1001),
-	}
-	if err := sendConn.Write(blockAnnouncement); err != nil {
-		t.Fatalf("could not write to connection: %v", err)
-	}
-
-	switch msg := receiveConn.ReadAndServe(s.chain).(type) {
-	case *NewBlock:
-		assert.Equal(t, blockAnnouncement.Block.Header(), msg.Block.Header(),
-			"wrong block header in announcement")
-		assert.Equal(t, blockAnnouncement.TD, msg.TD,
-			"wrong TD in announcement")
-	case *NewBlockHashes:
-		hashes := *msg
-		assert.Equal(t, blockAnnouncement.Block.Hash(), hashes[0].Hash,
-			"wrong block hash in announcement")
-	default:
-		t.Fatalf("unexpected: %#v", msg)
+		Block: s.fullChain.blocks[nextBlock],
+		TD:    s.fullChain.TD(nextBlock + 1),
 	}
+	s.testAnnounce(t, sendConn, receiveConn, blockAnnouncement)
 	// update test suite chain
-	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[1000])
+	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
 	// wait for client to update its chain
 	if err := receiveConn.waitForBlock(s.chain.Head()); err != nil {
 		t.Fatal(err)
 	}
 }
 
+// TestMaliciousHandshake tries to send malicious data during the handshake.
+func (s *Suite) TestMaliciousHandshake(t *utesting.T) {
+	conn, err := s.dial()
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	// write hello to client
+	pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:]
+	handshakes := []*Hello{
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: largeString(2), Version: 64},
+			},
+			ID: pub0,
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+			},
+			ID: append(pub0, byte(0)),
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+			},
+			ID: append(pub0, pub0...),
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+			},
+			ID: largeBuffer(2),
+		},
+		{
+			Version: 5,
+			Caps: []p2p.Cap{
+				{Name: largeString(2), Version: 64},
+			},
+			ID: largeBuffer(2),
+		},
+	}
+	for i, handshake := range handshakes {
+		t.Logf("Testing malicious handshake %v\n", i)
+		// Init the handshake
+		if err := conn.Write(handshake); err != nil {
+			t.Fatalf("could not write to connection: %v", err)
+		}
+		// check that the peer disconnected
+		timeout := 20 * time.Second
+		// Discard one hello
+		for i := 0; i < 2; i++ {
+			switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
+			case *Disconnect:
+			case *Error:
+			case *Hello:
+				// Hello's are send concurrently, so ignore them
+				continue
+			default:
+				t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+			}
+		}
+		// Dial for the next round
+		conn, err = s.dial()
+		if err != nil {
+			t.Fatalf("could not dial: %v", err)
+		}
+	}
+}
+
+// TestLargeAnnounce tests the announcement mechanism with a large block.
+func (s *Suite) TestLargeAnnounce(t *utesting.T) {
+	nextBlock := len(s.chain.blocks)
+	blocks := []*NewBlock{
+		{
+			Block: largeBlock(),
+			TD:    s.fullChain.TD(nextBlock + 1),
+		},
+		{
+			Block: s.fullChain.blocks[nextBlock],
+			TD:    largeNumber(2),
+		},
+		{
+			Block: largeBlock(),
+			TD:    largeNumber(2),
+		},
+		{
+			Block: s.fullChain.blocks[nextBlock],
+			TD:    s.fullChain.TD(nextBlock + 1),
+		},
+	}
+
+	for i, blockAnnouncement := range blocks[0:3] {
+		t.Logf("Testing malicious announcement: %v\n", i)
+		sendConn := s.setupConnection(t)
+		if err := sendConn.Write(blockAnnouncement); err != nil {
+			t.Fatalf("could not write to connection: %v", err)
+		}
+		// Invalid announcement, check that peer disconnected
+		switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) {
+		case *Disconnect:
+		case *Error:
+			break
+		default:
+			t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg))
+		}
+	}
+	// Test the last block as a valid block
+	sendConn := s.setupConnection(t)
+	receiveConn := s.setupConnection(t)
+	s.testAnnounce(t, sendConn, receiveConn, blocks[3])
+	// update test suite chain
+	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
+	// wait for client to update its chain
+	if err := receiveConn.waitForBlock(s.fullChain.blocks[nextBlock]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s *Suite) testAnnounce(t *utesting.T, sendConn, receiveConn *Conn, blockAnnouncement *NewBlock) {
+	// Announce the block.
+	if err := sendConn.Write(blockAnnouncement); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	s.waitAnnounce(t, receiveConn, blockAnnouncement)
+}
+
+func (s *Suite) waitAnnounce(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) {
+	timeout := 20 * time.Second
+	switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
+	case *NewBlock:
+		t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block))
+		assert.Equal(t,
+			blockAnnouncement.Block.Header(), msg.Block.Header(),
+			"wrong block header in announcement",
+		)
+		assert.Equal(t,
+			blockAnnouncement.TD, msg.TD,
+			"wrong TD in announcement",
+		)
+	case *NewBlockHashes:
+		message := *msg
+		t.Logf("received NewBlockHashes message: %s", pretty.Sdump(message))
+		assert.Equal(t, blockAnnouncement.Block.Hash(), message[0].Hash,
+			"wrong block hash in announcement",
+		)
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
+	}
+}
+
+func (s *Suite) setupConnection(t *utesting.T) *Conn {
+	// create conn
+	sendConn, err := s.dial()
+	if err != nil {
+		t.Fatalf("could not dial: %v", err)
+	}
+	sendConn.handshake(t)
+	sendConn.statusExchange(t, s.chain, nil)
+	return sendConn
+}
+
 // dial attempts to dial the given node and perform a handshake,
 // returning the created Conn if successful.
 func (s *Suite) dial() (*Conn, error) {
 	var conn Conn
-
+	// dial
 	fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", s.Dest.IP(), s.Dest.TCP()))
 	if err != nil {
 		return nil, err
 	}
 	conn.Conn = rlpx.NewConn(fd, s.Dest.Pubkey())
-
 	// do encHandshake
 	conn.ourKey, _ = crypto.GenerateKey()
 	_, err = conn.Handshake(conn.ourKey)
 	if err != nil {
 		return nil, err
 	}
-
+	// set default p2p capabilities
+	conn.caps = []p2p.Cap{
+		{Name: "eth", Version: 64},
+		{Name: "eth", Version: 65},
+	}
 	return &conn, nil
 }
+
+func (s *Suite) TestTransaction(t *utesting.T) {
+	tests := []*types.Transaction{
+		getNextTxFromChain(t, s),
+		unknownTx(t, s),
+	}
+	for i, tx := range tests {
+		t.Logf("Testing tx propagation: %v\n", i)
+		sendSuccessfulTx(t, s, tx)
+	}
+}
+
+func (s *Suite) TestMaliciousTx(t *utesting.T) {
+	tests := []*types.Transaction{
+		getOldTxFromChain(t, s),
+		invalidNonceTx(t, s),
+		hugeAmount(t, s),
+		hugeGasPrice(t, s),
+		hugeData(t, s),
+	}
+	for i, tx := range tests {
+		t.Logf("Testing malicious tx propagation: %v\n", i)
+		sendFailingTx(t, s, tx)
+	}
+}

cmd/devp2p/internal/ethtest/testdata/genesis.json

@@ -1,6 +1,6 @@
 {
     "config": {
-        "chainId": 1,
+        "chainId": 19763,
         "homesteadBlock": 0,
         "eip150Block": 0,
         "eip155Block": 0,
@@ -11,16 +11,16 @@
     "nonce": "0xdeadbeefdeadbeef",
     "timestamp": "0x0",
     "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000",
-    "gasLimit": "0x8000000",
-    "difficulty": "0x10",
+    "gasLimit": "0x80000000",
+    "difficulty": "0x20000",
     "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
     "coinbase": "0x0000000000000000000000000000000000000000",
     "alloc": {
         "71562b71999873db5b286df957af199ec94617f7": {
-            "balance": "0xf4240"
+            "balance": "0xffffffffffffffffffffffffff"
         }
     },
     "number": "0x0",
     "gasUsed": "0x0",
     "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000"
-}
+}

cmd/devp2p/internal/ethtest/transaction.go

@@ -0,0 +1,189 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethtest
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+)
+
+//var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7")
+var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+
+func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) {
+	sendConn := s.setupConnection(t)
+	sendSuccessfulTxWithConn(t, s, tx, sendConn)
+}
+
+func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, sendConn *Conn) {
+	t.Logf("sending tx: %v %v %v\n", tx.Hash().String(), tx.GasPrice(), tx.Gas())
+	// Send the transaction
+	if err := sendConn.Write(&Transactions{tx}); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(100 * time.Millisecond)
+	recvConn := s.setupConnection(t)
+	// Wait for the transaction announcement
+	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
+	case *Transactions:
+		recTxs := *msg
+		for _, gotTx := range recTxs {
+			if gotTx.Hash() == tx.Hash() {
+				// Ok
+				return
+			}
+		}
+		t.Fatalf("missing transaction: got %v missing %v", recTxs, tx.Hash())
+	case *NewPooledTransactionHashes:
+		txHashes := *msg
+		for _, gotHash := range txHashes {
+			if gotHash == tx.Hash() {
+				return
+			}
+		}
+		t.Fatalf("missing transaction announcement: got %v missing %v", txHashes, tx.Hash())
+	default:
+		t.Fatalf("unexpected message in sendSuccessfulTx: %s", pretty.Sdump(msg))
+	}
+}
+
+func sendFailingTx(t *utesting.T, s *Suite, tx *types.Transaction) {
+	sendConn, recvConn := s.setupConnection(t), s.setupConnection(t)
+	sendFailingTxWithConns(t, s, tx, sendConn, recvConn)
+}
+
+func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, sendConn, recvConn *Conn) {
+	// Wait for a transaction announcement
+	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
+	case *NewPooledTransactionHashes:
+		break
+	default:
+		t.Logf("unexpected message, logging: %v", pretty.Sdump(msg))
+	}
+	// Send the transaction
+	if err := sendConn.Write(&Transactions{tx}); err != nil {
+		t.Fatal(err)
+	}
+	// Wait for another transaction announcement
+	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
+	case *Transactions:
+		t.Fatalf("Received unexpected transaction announcement: %v", msg)
+	case *NewPooledTransactionHashes:
+		t.Fatalf("Received unexpected pooledTx announcement: %v", msg)
+	case *Error:
+		// Transaction should not be announced -> wait for timeout
+		return
+	default:
+		t.Fatalf("unexpected message in sendFailingTx: %s", pretty.Sdump(msg))
+	}
+}
+
+func unknownTx(t *utesting.T, s *Suite) *types.Transaction {
+	tx := getNextTxFromChain(t, s)
+	var to common.Address
+	if tx.To() != nil {
+		to = *tx.To()
+	}
+	txNew := types.NewTransaction(tx.Nonce()+1, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data())
+	return signWithFaucet(t, txNew)
+}
+
+func getNextTxFromChain(t *utesting.T, s *Suite) *types.Transaction {
+	// Get a new transaction
+	var tx *types.Transaction
+	for _, blocks := range s.fullChain.blocks[s.chain.Len():] {
+		txs := blocks.Transactions()
+		if txs.Len() != 0 {
+			tx = txs[0]
+			break
+		}
+	}
+	if tx == nil {
+		t.Fatal("could not find transaction")
+	}
+	return tx
+}
+
+func getOldTxFromChain(t *utesting.T, s *Suite) *types.Transaction {
+	var tx *types.Transaction
+	for _, blocks := range s.fullChain.blocks[:s.chain.Len()-1] {
+		txs := blocks.Transactions()
+		if txs.Len() != 0 {
+			tx = txs[0]
+			break
+		}
+	}
+	if tx == nil {
+		t.Fatal("could not find transaction")
+	}
+	return tx
+}
+
+func invalidNonceTx(t *utesting.T, s *Suite) *types.Transaction {
+	tx := getNextTxFromChain(t, s)
+	var to common.Address
+	if tx.To() != nil {
+		to = *tx.To()
+	}
+	txNew := types.NewTransaction(tx.Nonce()-2, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data())
+	return signWithFaucet(t, txNew)
+}
+
+func hugeAmount(t *utesting.T, s *Suite) *types.Transaction {
+	tx := getNextTxFromChain(t, s)
+	amount := largeNumber(2)
+	var to common.Address
+	if tx.To() != nil {
+		to = *tx.To()
+	}
+	txNew := types.NewTransaction(tx.Nonce(), to, amount, tx.Gas(), tx.GasPrice(), tx.Data())
+	return signWithFaucet(t, txNew)
+}
+
+func hugeGasPrice(t *utesting.T, s *Suite) *types.Transaction {
+	tx := getNextTxFromChain(t, s)
+	gasPrice := largeNumber(2)
+	var to common.Address
+	if tx.To() != nil {
+		to = *tx.To()
+	}
+	txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), gasPrice, tx.Data())
+	return signWithFaucet(t, txNew)
+}
+
+func hugeData(t *utesting.T, s *Suite) *types.Transaction {
+	tx := getNextTxFromChain(t, s)
+	var to common.Address
+	if tx.To() != nil {
+		to = *tx.To()
+	}
+	txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), tx.GasPrice(), largeBuffer(2))
+	return signWithFaucet(t, txNew)
+}
+
+func signWithFaucet(t *utesting.T, tx *types.Transaction) *types.Transaction {
+	signer := types.HomesteadSigner{}
+	signedTx, err := types.SignTx(tx, signer, faucetKey)
+	if err != nil {
+		t.Fatalf("could not sign tx: %v\n", err)
+	}
+	return signedTx
+}

cmd/devp2p/internal/ethtest/types.go

@@ -19,15 +19,12 @@ package ethtest
 import (
 	"crypto/ecdsa"
 	"fmt"
-	"io"
-	"math/big"
 	"reflect"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/forkid"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	"github.com/ethereum/go-ethereum/internal/utesting"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/rlpx"
@@ -42,10 +39,14 @@ type Error struct {
 	err error
 }
 
-func (e *Error) Unwrap() error    { return e.err }
-func (e *Error) Error() string    { return e.err.Error() }
-func (e *Error) Code() int        { return -1 }
-func (e *Error) GoString() string { return e.Error() }
+func (e *Error) Unwrap() error  { return e.err }
+func (e *Error) Error() string  { return e.err.Error() }
+func (e *Error) Code() int      { return -1 }
+func (e *Error) String() string { return e.Error() }
+
+func errorf(format string, args ...interface{}) *Error {
+	return &Error{fmt.Errorf(format, args...)}
+}
 
 // Hello is the RLP structure of the protocol handshake.
 type Hello struct {
@@ -77,104 +78,60 @@ type Pong struct{}
 func (p Pong) Code() int { return 0x03 }
 
 // Status is the network packet for the status message for eth/64 and later.
-type Status struct {
-	ProtocolVersion uint32
-	NetworkID       uint64
-	TD              *big.Int
-	Head            common.Hash
-	Genesis         common.Hash
-	ForkID          forkid.ID
-}
+type Status eth.StatusPacket
 
 func (s Status) Code() int { return 16 }
 
 // NewBlockHashes is the network packet for the block announcements.
-type NewBlockHashes []struct {
-	Hash   common.Hash // Hash of one particular block being announced
-	Number uint64      // Number of one particular block being announced
-}
+type NewBlockHashes eth.NewBlockHashesPacket
 
 func (nbh NewBlockHashes) Code() int { return 17 }
 
-// NewBlock is the network packet for the block propagation message.
-type NewBlock struct {
-	Block *types.Block
-	TD    *big.Int
-}
+type Transactions eth.TransactionsPacket
 
-func (nb NewBlock) Code() int { return 23 }
+func (t Transactions) Code() int { return 18 }
 
 // GetBlockHeaders represents a block header query.
-type GetBlockHeaders struct {
-	Origin  hashOrNumber // Block from which to retrieve headers
-	Amount  uint64       // Maximum number of headers to retrieve
-	Skip    uint64       // Blocks to skip between consecutive headers
-	Reverse bool         // Query direction (false = rising towards latest, true = falling towards genesis)
-}
+type GetBlockHeaders eth.GetBlockHeadersPacket
 
 func (g GetBlockHeaders) Code() int { return 19 }
 
-type BlockHeaders []*types.Header
+type BlockHeaders eth.BlockHeadersPacket
 
 func (bh BlockHeaders) Code() int { return 20 }
 
-// HashOrNumber is a combined field for specifying an origin block.
-type hashOrNumber struct {
-	Hash   common.Hash // Block hash from which to retrieve headers (excludes Number)
-	Number uint64      // Block hash from which to retrieve headers (excludes Hash)
-}
-
-// EncodeRLP is a specialized encoder for hashOrNumber to encode only one of the
-// two contained union fields.
-func (hn *hashOrNumber) EncodeRLP(w io.Writer) error {
-	if hn.Hash == (common.Hash{}) {
-		return rlp.Encode(w, hn.Number)
-	}
-	if hn.Number != 0 {
-		return fmt.Errorf("both origin hash (%x) and number (%d) provided", hn.Hash, hn.Number)
-	}
-	return rlp.Encode(w, hn.Hash)
-}
-
-// DecodeRLP is a specialized decoder for hashOrNumber to decode the contents
-// into either a block hash or a block number.
-func (hn *hashOrNumber) DecodeRLP(s *rlp.Stream) error {
-	_, size, _ := s.Kind()
-	origin, err := s.Raw()
-	if err == nil {
-		switch {
-		case size == 32:
-			err = rlp.DecodeBytes(origin, &hn.Hash)
-		case size <= 8:
-			err = rlp.DecodeBytes(origin, &hn.Number)
-		default:
-			err = fmt.Errorf("invalid input size %d for origin", size)
-		}
-	}
-	return err
-}
-
 // GetBlockBodies represents a GetBlockBodies request
-type GetBlockBodies []common.Hash
+type GetBlockBodies eth.GetBlockBodiesPacket
 
 func (gbb GetBlockBodies) Code() int { return 21 }
 
 // BlockBodies is the network packet for block content distribution.
-type BlockBodies []*types.Body
+type BlockBodies eth.BlockBodiesPacket
 
 func (bb BlockBodies) Code() int { return 22 }
 
+// NewBlock is the network packet for the block propagation message.
+type NewBlock eth.NewBlockPacket
+
+func (nb NewBlock) Code() int { return 23 }
+
+// NewPooledTransactionHashes is the network packet for the tx hash propagation message.
+type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket
+
+func (nb NewPooledTransactionHashes) Code() int { return 24 }
+
 // Conn represents an individual connection with a peer
 type Conn struct {
 	*rlpx.Conn
 	ourKey             *ecdsa.PrivateKey
 	ethProtocolVersion uint
+	caps               []p2p.Cap
 }
 
 func (c *Conn) Read() Message {
 	code, rawData, _, err := c.Conn.Read()
 	if err != nil {
-		return &Error{fmt.Errorf("could not read from connection: %v", err)}
+		return errorf("could not read from connection: %v", err)
 	}
 
 	var msg Message
@@ -201,21 +158,27 @@ func (c *Conn) Read() Message {
 		msg = new(NewBlock)
 	case (NewBlockHashes{}).Code():
 		msg = new(NewBlockHashes)
+	case (Transactions{}).Code():
+		msg = new(Transactions)
+	case (NewPooledTransactionHashes{}).Code():
+		msg = new(NewPooledTransactionHashes)
 	default:
-		return &Error{fmt.Errorf("invalid message code: %d", code)}
+		return errorf("invalid message code: %d", code)
 	}
-
+	// if message is devp2p, decode here
 	if err := rlp.DecodeBytes(rawData, msg); err != nil {
-		return &Error{fmt.Errorf("could not rlp decode message: %v", err)}
+		return errorf("could not rlp decode message: %v", err)
 	}
-
 	return msg
 }
 
 // ReadAndServe serves GetBlockHeaders requests while waiting
 // on another message from the node.
-func (c *Conn) ReadAndServe(chain *Chain) Message {
-	for {
+func (c *Conn) ReadAndServe(chain *Chain, timeout time.Duration) Message {
+	start := time.Now()
+	for time.Since(start) < timeout {
+		timeout := time.Now().Add(10 * time.Second)
+		c.SetReadDeadline(timeout)
 		switch msg := c.Read().(type) {
 		case *Ping:
 			c.Write(&Pong{})
@@ -223,51 +186,55 @@ func (c *Conn) ReadAndServe(chain *Chain) Message {
 			req := *msg
 			headers, err := chain.GetHeaders(req)
 			if err != nil {
-				return &Error{fmt.Errorf("could not get headers for inbound header request: %v", err)}
+				return errorf("could not get headers for inbound header request: %v", err)
 			}
 
 			if err := c.Write(headers); err != nil {
-				return &Error{fmt.Errorf("could not write to connection: %v", err)}
+				return errorf("could not write to connection: %v", err)
 			}
 		default:
 			return msg
 		}
 	}
+	return errorf("no message received within %v", timeout)
 }
 
 func (c *Conn) Write(msg Message) error {
-	payload, err := rlp.EncodeToBytes(msg)
+	// check if message is eth protocol message
+	var (
+		payload []byte
+		err     error
+	)
+	payload, err = rlp.EncodeToBytes(msg)
 	if err != nil {
 		return err
 	}
 	_, err = c.Conn.Write(uint64(msg.Code()), payload)
 	return err
-
 }
 
 // handshake checks to make sure a `HELLO` is received.
 func (c *Conn) handshake(t *utesting.T) Message {
-	// write protoHandshake to client
+	defer c.SetDeadline(time.Time{})
+	c.SetDeadline(time.Now().Add(10 * time.Second))
+
+	// write hello to client
 	pub0 := crypto.FromECDSAPub(&c.ourKey.PublicKey)[1:]
 	ourHandshake := &Hello{
 		Version: 5,
-		Caps: []p2p.Cap{
-			{Name: "eth", Version: 64},
-			{Name: "eth", Version: 65},
-		},
-		ID: pub0,
+		Caps:    c.caps,
+		ID:      pub0,
 	}
 	if err := c.Write(ourHandshake); err != nil {
 		t.Fatalf("could not write to connection: %v", err)
 	}
-	// read protoHandshake from client
+	// read hello from client
 	switch msg := c.Read().(type) {
 	case *Hello:
 		// set snappy if version is at least 5
 		if msg.Version >= 5 {
 			c.SetSnappy(true)
 		}
-
 		c.negotiateEthProtocol(msg.Caps)
 		if c.ethProtocolVersion == 0 {
 			t.Fatalf("unexpected eth protocol version")
@@ -296,16 +263,18 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) {
 
 // statusExchange performs a `Status` message exchange with the given
 // node.
-func (c *Conn) statusExchange(t *utesting.T, chain *Chain) Message {
+func (c *Conn) statusExchange(t *utesting.T, chain *Chain, status *Status) Message {
+	defer c.SetDeadline(time.Time{})
+	c.SetDeadline(time.Now().Add(20 * time.Second))
+
 	// read status message from client
 	var message Message
-
 loop:
 	for {
 		switch msg := c.Read().(type) {
 		case *Status:
 			if msg.Head != chain.blocks[chain.Len()-1].Hash() {
-				t.Fatalf("wrong head in status: %v", msg.Head)
+				t.Fatalf("wrong head block in status: %s", msg.Head.String())
 			}
 			if msg.TD.Cmp(chain.TD(chain.Len())) != 0 {
 				t.Fatalf("wrong TD in status: %v", msg.TD)
@@ -321,22 +290,25 @@ loop:
 			c.Write(&Pong{}) // TODO (renaynay): in the future, this should be an error
 			// (PINGs should not be a response upon fresh connection)
 		default:
-			t.Fatalf("bad status message: %#v", msg)
+			t.Fatalf("bad status message: %s", pretty.Sdump(msg))
 		}
 	}
 	// make sure eth protocol version is set for negotiation
 	if c.ethProtocolVersion == 0 {
 		t.Fatalf("eth protocol version must be set in Conn")
 	}
-	// write status message to client
-	status := Status{
-		ProtocolVersion: uint32(c.ethProtocolVersion),
-		NetworkID:       1,
-		TD:              chain.TD(chain.Len()),
-		Head:            chain.blocks[chain.Len()-1].Hash(),
-		Genesis:         chain.blocks[0].Hash(),
-		ForkID:          chain.ForkID(),
+	if status == nil {
+		// write status message to client
+		status = &Status{
+			ProtocolVersion: uint32(c.ethProtocolVersion),
+			NetworkID:       chain.chainConfig.ChainID.Uint64(),
+			TD:              chain.TD(chain.Len()),
+			Head:            chain.blocks[chain.Len()-1].Hash(),
+			Genesis:         chain.blocks[0].Hash(),
+			ForkID:          chain.ForkID(),
+		}
 	}
+
 	if err := c.Write(status); err != nil {
 		t.Fatalf("could not write to connection: %v", err)
 	}
@@ -347,12 +319,15 @@ loop:
 // waitForBlock waits for confirmation from the client that it has
 // imported the given block.
 func (c *Conn) waitForBlock(block *types.Block) error {
+	defer c.SetReadDeadline(time.Time{})
+
+	timeout := time.Now().Add(20 * time.Second)
+	c.SetReadDeadline(timeout)
 	for {
-		req := &GetBlockHeaders{Origin: hashOrNumber{Hash: block.Hash()}, Amount: 1}
+		req := &GetBlockHeaders{Origin: eth.HashOrNumber{Hash: block.Hash()}, Amount: 1}
 		if err := c.Write(req); err != nil {
 			return err
 		}
-
 		switch msg := c.Read().(type) {
 		case *BlockHeaders:
 			if len(*msg) > 0 {
@@ -360,7 +335,7 @@ func (c *Conn) waitForBlock(block *types.Block) error {
 			}
 			time.Sleep(100 * time.Millisecond)
 		default:
-			return fmt.Errorf("invalid message: %v", msg)
+			return fmt.Errorf("invalid message: %s", pretty.Sdump(msg))
 		}
 	}
 }

cmd/devp2p/nodesetcmd.go

@@ -95,6 +95,7 @@ var filterFlags = map[string]nodeFilterC{
 	"-min-age":     {1, minAgeFilter},
 	"-eth-network": {1, ethFilter},
 	"-les-server":  {0, lesFilter},
+	"-snap":        {0, snapFilter},
 }
 
 func parseFilters(args []string) ([]nodeFilter, error) {
@@ -104,15 +105,15 @@ func parseFilters(args []string) ([]nodeFilter, error) {
 		if !ok {
 			return nil, fmt.Errorf("invalid filter %q", args[0])
 		}
-		if len(args) < fc.narg {
-			return nil, fmt.Errorf("filter %q wants %d arguments, have %d", args[0], fc.narg, len(args))
+		if len(args)-1 < fc.narg {
+			return nil, fmt.Errorf("filter %q wants %d arguments, have %d", args[0], fc.narg, len(args)-1)
 		}
-		filter, err := fc.fn(args[1:])
+		filter, err := fc.fn(args[1 : 1+fc.narg])
 		if err != nil {
 			return nil, fmt.Errorf("%s: %v", args[0], err)
 		}
 		filters = append(filters, filter)
-		args = args[fc.narg+1:]
+		args = args[1+fc.narg:]
 	}
 	return filters, nil
 }
@@ -191,3 +192,13 @@ func lesFilter(args []string) (nodeFilter, error) {
 	}
 	return f, nil
 }
+
+func snapFilter(args []string) (nodeFilter, error) {
+	f := func(n nodeJSON) bool {
+		var snap struct {
+			_ []rlp.RawValue `rlp:"tail"`
+		}
+		return n.N.Load(enr.WithEntry("snap", &snap)) == nil
+	}
+	return f, nil
+}

cmd/devp2p/rlpxcmd.go

@@ -19,11 +19,9 @@ package main
 import (
 	"fmt"
 	"net"
-	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/internal/utesting"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/rlpx"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -47,9 +45,12 @@ var (
 	rlpxEthTestCommand = cli.Command{
 		Name:      "eth-test",
 		Usage:     "Runs tests against a node",
-		ArgsUsage: "<node> <path_to_chain.rlp_file>",
+		ArgsUsage: "<node> <chain.rlp> <genesis.json>",
 		Action:    rlpxEthTest,
-		Flags:     []cli.Flag{testPatternFlag},
+		Flags: []cli.Flag{
+			testPatternFlag,
+			testTAPFlag,
+		},
 	}
 )
 
@@ -88,22 +89,14 @@ func rlpxPing(ctx *cli.Context) error {
 	return nil
 }
 
+// rlpxEthTest runs the eth protocol test suite.
 func rlpxEthTest(ctx *cli.Context) error {
 	if ctx.NArg() < 3 {
 		exit("missing path to chain.rlp as command-line argument")
 	}
-
-	suite := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2])
-
-	// Filter and run test cases.
-	tests := suite.AllTests()
-	if ctx.IsSet(testPatternFlag.Name) {
-		tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
+	suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2])
+	if err != nil {
+		exit(err)
 	}
-	results := utesting.RunTests(tests, os.Stdout)
-	if fails := utesting.CountFailures(results); fails > 0 {
-		return fmt.Errorf("%v of %v tests passed.", len(tests)-fails, len(tests))
-	}
-	fmt.Printf("all tests passed\n")
-	return nil
+	return runTests(ctx, suite.EthTests())
 }

cmd/devp2p/runtest.go

@@ -0,0 +1,69 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/log"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var (
+	testPatternFlag = cli.StringFlag{
+		Name:  "run",
+		Usage: "Pattern of test suite(s) to run",
+	}
+	testTAPFlag = cli.BoolFlag{
+		Name:  "tap",
+		Usage: "Output TAP",
+	}
+	// These two are specific to the discovery tests.
+	testListen1Flag = cli.StringFlag{
+		Name:  "listen1",
+		Usage: "IP address of the first tester",
+		Value: v4test.Listen1,
+	}
+	testListen2Flag = cli.StringFlag{
+		Name:  "listen2",
+		Usage: "IP address of the second tester",
+		Value: v4test.Listen2,
+	}
+)
+
+func runTests(ctx *cli.Context, tests []utesting.Test) error {
+	// Filter test cases.
+	if ctx.IsSet(testPatternFlag.Name) {
+		tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
+	}
+	// Disable logging unless explicitly enabled.
+	if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") {
+		log.Root().SetHandler(log.DiscardHandler())
+	}
+	// Run the tests.
+	var run = utesting.RunTests
+	if ctx.Bool(testTAPFlag.Name) {
+		run = utesting.RunTAP
+	}
+	results := run(tests, os.Stdout)
+	if utesting.CountFailures(results) > 0 {
+		os.Exit(1)
+	}
+	return nil
+}

cmd/ethkey/generate.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -86,9 +86,12 @@ If you want to encrypt an existing private key, it can be specified by setting
 		}
 
 		// Create the keyfile object with a random UUID.
-		id := uuid.NewRandom()
+		UUID, err := uuid.NewRandom()
+		if err != nil {
+			utils.Fatalf("Failed to generate random uuid: %v", err)
+		}
 		key := &keystore.Key{
-			Id:         id,
+			Id:         UUID,
 			Address:    crypto.PubkeyToAddress(privateKey.PublicKey),
 			PrivateKey: privateKey,
 		}

cmd/evm/README.md

@@ -30,7 +30,7 @@ Command line params that has to be supported are
    --trace.nomemory                   Disable full memory dump in traces
    --trace.nostack                    Disable stack output in traces
    --trace.noreturndata               Disable return data output in traces
-   --output.basedir value             Specifies where output files are placed. Will be created if it does not exist. (default: ".")
+   --output.basedir value             Specifies where output files are placed. Will be created if it does not exist.
    --output.alloc alloc               Determines where to put the alloc of the post-state.
                                       `stdout` - into the stdout output
                                       `stderr` - into the stderr output
@@ -237,10 +237,10 @@ Example where blockhashes are provided:
 cat trace-0-0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81.jsonl | grep BLOCKHASH -C2
 ```
 ```
-{"pc":0,"op":96,"gas":"0x5f58ef8","gasCost":"0x3","memory":"0x","memSize":0,"stack":[],"returnStack":[],"returnData":null,"depth":1,"refund":0,"opName":"PUSH1","error":""}
-{"pc":2,"op":64,"gas":"0x5f58ef5","gasCost":"0x14","memory":"0x","memSize":0,"stack":["0x1"],"returnStack":[],"returnData":null,"depth":1,"refund":0,"opName":"BLOCKHASH","error":""}
-{"pc":3,"op":0,"gas":"0x5f58ee1","gasCost":"0x0","memory":"0x","memSize":0,"stack":["0xdac58aa524e50956d0c0bae7f3f8bb9d35381365d07804dd5b48a5a297c06af4"],"returnStack":[],"returnData":null,"depth":1,"refund":0,"opName":"STOP","error":""}
-{"output":"","gasUsed":"0x17","time":112885}
+{"pc":0,"op":96,"gas":"0x5f58ef8","gasCost":"0x3","memory":"0x","memSize":0,"stack":[],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"PUSH1","error":""}
+{"pc":2,"op":64,"gas":"0x5f58ef5","gasCost":"0x14","memory":"0x","memSize":0,"stack":["0x1"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"BLOCKHASH","error":""}
+{"pc":3,"op":0,"gas":"0x5f58ee1","gasCost":"0x0","memory":"0x","memSize":0,"stack":["0xdac58aa524e50956d0c0bae7f3f8bb9d35381365d07804dd5b48a5a297c06af4"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"STOP","error":""}
+{"output":"","gasUsed":"0x17","time":142709}
 ```
 
 In this example, the caller has not provided the required blockhash:
@@ -256,9 +256,9 @@ Error code: 4
 Another thing that can be done, is to chain invocations:
 ```
 ./evm t8n --input.alloc=./testdata/1/alloc.json --input.txs=./testdata/1/txs.json --input.env=./testdata/1/env.json --output.alloc=stdout | ./evm t8n --input.alloc=stdin --input.env=./testdata/1/env.json --input.txs=./testdata/1/txs.json
-INFO [08-03|15:25:15.168] rejected tx                              index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low"
-INFO [08-03|15:25:15.169] rejected tx                              index=0 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low"
-INFO [08-03|15:25:15.169] rejected tx                              index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low"
+INFO [01-21|22:41:22.963] rejected tx                              index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
+INFO [01-21|22:41:22.966] rejected tx                              index=0 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
+INFO [01-21|22:41:22.967] rejected tx                              index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
 
 ```
 What happened here, is that we first applied two identical transactions, so the second one was rejected. 
@@ -267,4 +267,3 @@ the same two transactions: this time, both failed due to too low nonce.
 
 In order to meaningfully chain invocations, one would need to provide meaningful new `env`, otherwise the
 actual blocknumber (exposed to the EVM) would not increase.
-

cmd/evm/compiler.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
 
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 )
 
 var compileCommand = cli.Command{

cmd/evm/disasm.go

@@ -23,7 +23,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/core/asm"
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 )
 
 var disasmCommand = cli.Command{

cmd/evm/internal/t8ntool/execution.go

@@ -110,7 +110,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		txIndex     = 0
 	)
 	gaspool.AddGas(pre.Env.GasLimit)
-	vmContext := vm.Context{
+	vmContext := vm.BlockContext{
 		CanTransfer: core.CanTransfer,
 		Transfer:    core.Transfer,
 		Coinbase:    pre.Env.Coinbase,
@@ -119,7 +119,6 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		Difficulty:  pre.Env.Difficulty,
 		GasLimit:    pre.Env.GasLimit,
 		GetHash:     getHash,
-		// GasPrice and Origin needs to be set per transaction
 	}
 	// If DAO is supported/enabled, we need to handle it here. In geth 'proper', it's
 	// done in StateProcessor.Process(block, ...), right before transactions are applied.
@@ -143,11 +142,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		vmConfig.Tracer = tracer
 		vmConfig.Debug = (tracer != nil)
 		statedb.Prepare(tx.Hash(), blockHash, txIndex)
-		vmContext.GasPrice = msg.GasPrice()
-		vmContext.Origin = msg.From()
-
-		evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig)
+		txContext := core.NewEVMTxContext(msg)
 		snapshot := statedb.Snapshot()
+		evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig)
+
 		// (ret []byte, usedGas uint64, failed bool, err error)
 		msgResult, err := core.ApplyMessage(evm, msg, gaspool)
 		if err != nil {
@@ -161,7 +159,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			return nil, nil, NewError(ErrorMissingBlockhash, hashError)
 		}
 		gasUsed += msgResult.UsedGas
-		// Create a new receipt for the transaction, storing the intermediate root and gas used by the tx
+
+		// Receipt:
 		{
 			var root []byte
 			if chainConfig.IsByzantium(vmContext.BlockNumber) {
@@ -170,22 +169,32 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 				root = statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber)).Bytes()
 			}
 
-			receipt := types.NewReceipt(root, msgResult.Failed(), gasUsed)
+			// Create a new receipt for the transaction, storing the intermediate root and
+			// gas used by the tx.
+			receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: gasUsed}
+			if msgResult.Failed() {
+				receipt.Status = types.ReceiptStatusFailed
+			} else {
+				receipt.Status = types.ReceiptStatusSuccessful
+			}
 			receipt.TxHash = tx.Hash()
 			receipt.GasUsed = msgResult.UsedGas
-			// if the transaction created a contract, store the creation address in the receipt.
+
+			// If the transaction created a contract, store the creation address in the receipt.
 			if msg.To() == nil {
-				receipt.ContractAddress = crypto.CreateAddress(evm.Context.Origin, tx.Nonce())
+				receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce())
 			}
-			// Set the receipt logs and create a bloom for filtering
+
+			// Set the receipt logs and create the bloom filter.
 			receipt.Logs = statedb.GetLogs(tx.Hash())
 			receipt.Bloom = types.CreateBloom(types.Receipts{receipt})
-			// These three are non-consensus fields
+			// These three are non-consensus fields:
 			//receipt.BlockHash
-			//receipt.BlockNumber =
+			//receipt.BlockNumber
 			receipt.TransactionIndex = uint(txIndex)
 			receipts = append(receipts, receipt)
 		}
+
 		txIndex++
 	}
 	statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber))
@@ -221,8 +230,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	}
 	execRs := &ExecutionResult{
 		StateRoot:   root,
-		TxRoot:      types.DeriveSha(includedTxs, new(trie.Trie)),
-		ReceiptRoot: types.DeriveSha(receipts, new(trie.Trie)),
+		TxRoot:      types.DeriveSha(includedTxs, trie.NewStackTrie(nil)),
+		ReceiptRoot: types.DeriveSha(receipts, trie.NewStackTrie(nil)),
 		Bloom:       types.CreateBloom(receipts),
 		LogsHash:    rlpHash(statedb.Logs()),
 		Receipts:    receipts,

cmd/evm/internal/t8ntool/flags.go

@@ -47,6 +47,11 @@ var (
 		Usage: "Specifies where output files are placed. Will be created if it does not exist.",
 		Value: "",
 	}
+	OutputBodyFlag = cli.StringFlag{
+		Name:  "output.body",
+		Usage: "If set, the RLP of the transactions (block body) will be written to this file.",
+		Value: "",
+	}
 	OutputAllocFlag = cli.StringFlag{
 		Name: "output.alloc",
 		Usage: "Determines where to put the `alloc` of the post-state.\n" +

cmd/evm/internal/t8ntool/transition.go

@@ -17,6 +17,7 @@
 package t8ntool
 
 import (
+	"crypto/ecdsa"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -25,12 +26,15 @@ import (
 	"path"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/tests"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -64,9 +68,9 @@ func (n *NumberedError) Code() int {
 }
 
 type input struct {
-	Alloc core.GenesisAlloc  `json:"alloc,omitempty"`
-	Env   *stEnv             `json:"env,omitempty"`
-	Txs   types.Transactions `json:"txs,omitempty"`
+	Alloc core.GenesisAlloc `json:"alloc,omitempty"`
+	Env   *stEnv            `json:"env,omitempty"`
+	Txs   []*txWithKey      `json:"txs,omitempty"`
 }
 
 func Main(ctx *cli.Context) error {
@@ -135,7 +139,7 @@ func Main(ctx *cli.Context) error {
 		txStr     = ctx.String(InputTxsFlag.Name)
 		inputData = &input{}
 	)
-
+	// Figure out the prestate alloc
 	if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector {
 		decoder := json.NewDecoder(os.Stdin)
 		decoder.Decode(inputData)
@@ -151,7 +155,9 @@ func Main(ctx *cli.Context) error {
 			return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling alloc-file: %v", err))
 		}
 	}
+	prestate.Pre = inputData.Alloc
 
+	// Set the block environment
 	if envStr != stdinSelector {
 		inFile, err := os.Open(envStr)
 		if err != nil {
@@ -165,26 +171,8 @@ func Main(ctx *cli.Context) error {
 		}
 		inputData.Env = &env
 	}
-
-	if txStr != stdinSelector {
-		inFile, err := os.Open(txStr)
-		if err != nil {
-			return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
-		}
-		defer inFile.Close()
-		decoder := json.NewDecoder(inFile)
-		var txs types.Transactions
-		if err := decoder.Decode(&txs); err != nil {
-			return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling txs-file: %v", err))
-		}
-		inputData.Txs = txs
-	}
-
-	prestate.Pre = inputData.Alloc
 	prestate.Env = *inputData.Env
-	txs = inputData.Txs
 
-	// Iterate over all the tests, run them and aggregate the results
 	vmConfig := vm.Config{
 		Tracer: tracer,
 		Debug:  (tracer != nil),
@@ -200,19 +188,107 @@ func Main(ctx *cli.Context) error {
 	// Set the chain id
 	chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name))
 
+	var txsWithKeys []*txWithKey
+	if txStr != stdinSelector {
+		inFile, err := os.Open(txStr)
+		if err != nil {
+			return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
+		}
+		defer inFile.Close()
+		decoder := json.NewDecoder(inFile)
+		if err := decoder.Decode(&txsWithKeys); err != nil {
+			return NewError(ErrorJson, fmt.Errorf("Failed unmarshaling txs-file: %v", err))
+		}
+	} else {
+		txsWithKeys = inputData.Txs
+	}
+	// We may have to sign the transactions.
+	signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number)))
+
+	if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil {
+		return NewError(ErrorJson, fmt.Errorf("Failed signing transactions: %v", err))
+	}
+
+	// Iterate over all the tests, run them and aggregate the results
+
 	// Run the test and aggregate the result
 	state, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer)
 	if err != nil {
 		return err
 	}
+	body, _ := rlp.EncodeToBytes(txs)
 	// Dump the excution result
-	//postAlloc := state.DumpGenesisFormat(false, false, false)
 	collector := make(Alloc)
 	state.DumpToCollector(collector, false, false, false, nil, -1)
-	return dispatchOutput(ctx, baseDir, result, collector)
+	return dispatchOutput(ctx, baseDir, result, collector, body)
 
 }
 
+// txWithKey is a helper-struct, to allow us to use the types.Transaction along with
+// a `secretKey`-field, for input
+type txWithKey struct {
+	key *ecdsa.PrivateKey
+	tx  *types.Transaction
+}
+
+func (t *txWithKey) UnmarshalJSON(input []byte) error {
+	// Read the secretKey, if present
+	type sKey struct {
+		Key *common.Hash `json:"secretKey"`
+	}
+	var key sKey
+	if err := json.Unmarshal(input, &key); err != nil {
+		return err
+	}
+	if key.Key != nil {
+		k := key.Key.Hex()[2:]
+		if ecdsaKey, err := crypto.HexToECDSA(k); err != nil {
+			return err
+		} else {
+			t.key = ecdsaKey
+		}
+	}
+	// Now, read the transaction itself
+	var tx types.Transaction
+	if err := json.Unmarshal(input, &tx); err != nil {
+		return err
+	}
+	t.tx = &tx
+	return nil
+}
+
+// signUnsignedTransactions converts the input txs to canonical transactions.
+//
+// The transactions can have two forms, either
+//   1. unsigned or
+//   2. signed
+// For (1), r, s, v, need so be zero, and the `secretKey` needs to be set.
+// If so, we sign it here and now, with the given `secretKey`
+// If the condition above is not met, then it's considered a signed transaction.
+//
+// To manage this, we read the transactions twice, first trying to read the secretKeys,
+// and secondly to read them with the standard tx json format
+func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) {
+	var signedTxs []*types.Transaction
+	for i, txWithKey := range txs {
+		tx := txWithKey.tx
+		key := txWithKey.key
+		v, r, s := tx.RawSignatureValues()
+		if key != nil && v.BitLen()+r.BitLen()+s.BitLen() == 0 {
+			// This transaction needs to be signed
+			signed, err := types.SignTx(tx, signer, key)
+			if err != nil {
+				return nil, NewError(ErrorJson, fmt.Errorf("Tx %d: failed to sign tx: %v", i, err))
+			}
+			signedTxs = append(signedTxs, signed)
+		} else {
+			// Already signed
+			signedTxs = append(signedTxs, tx)
+		}
+	}
+	return signedTxs, nil
+}
+
 type Alloc map[common.Address]core.GenesisAccount
 
 func (g Alloc) OnRoot(common.Hash) {}
@@ -241,15 +317,17 @@ func saveFile(baseDir, filename string, data interface{}) error {
 	if err != nil {
 		return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err))
 	}
-	if err = ioutil.WriteFile(path.Join(baseDir, filename), b, 0644); err != nil {
+	location := path.Join(baseDir, filename)
+	if err = ioutil.WriteFile(location, b, 0644); err != nil {
 		return NewError(ErrorIO, fmt.Errorf("failed writing output: %v", err))
 	}
+	log.Info("Wrote file", "file", location)
 	return nil
 }
 
 // dispatchOutput writes the output data to either stderr or stdout, or to the specified
 // files
-func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc) error {
+func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes) error {
 	stdOutObject := make(map[string]interface{})
 	stdErrObject := make(map[string]interface{})
 	dispatch := func(baseDir, fName, name string, obj interface{}) error {
@@ -258,6 +336,8 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a
 			stdOutObject[name] = obj
 		case "stderr":
 			stdErrObject[name] = obj
+		case "":
+			// don't save
 		default: // save to file
 			if err := saveFile(baseDir, fName, obj); err != nil {
 				return err
@@ -271,6 +351,9 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a
 	if err := dispatch(baseDir, ctx.String(OutputResultFlag.Name), "result", result); err != nil {
 		return err
 	}
+	if err := dispatch(baseDir, ctx.String(OutputBodyFlag.Name), "body", body); err != nil {
+		return err
+	}
 	if len(stdOutObject) > 0 {
 		b, err := json.MarshalIndent(stdOutObject, "", " ")
 		if err != nil {

cmd/evm/main.go

@@ -149,6 +149,7 @@ var stateTransitionCommand = cli.Command{
 		t8ntool.OutputBasedir,
 		t8ntool.OutputAllocFlag,
 		t8ntool.OutputResultFlag,
+		t8ntool.OutputBodyFlag,
 		t8ntool.InputAllocFlag,
 		t8ntool.InputEnvFlag,
 		t8ntool.InputTxsFlag,

cmd/evm/runner.go

@@ -38,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 )
 
 var runCommand = cli.Command{

cmd/evm/staterunner.go

@@ -28,7 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 )
 
 var stateTestCommand = cli.Command{

cmd/evm/testdata/8/alloc.json

@@ -0,0 +1,11 @@
+{
+  "0x000000000000000000000000000000000000aaaa": {
+    "balance": "0x03",
+    "code": "0x5854505854",
+    "nonce": "0x1"
+  },
+  "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+    "balance": "0x100000",
+    "nonce": "0x00"
+  }
+}

cmd/evm/testdata/8/env.json

@@ -0,0 +1,7 @@
+{
+  "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+  "currentDifficulty": "0x20000",
+  "currentGasLimit": "0x1000000000",
+  "currentNumber": "0x1000000",
+  "currentTimestamp": "0x04"
+}

cmd/evm/testdata/8/readme.md

@@ -0,0 +1,63 @@
+## EIP-2930 testing
+
+This test contains testcases for EIP-2930, which uses transactions with access lists. 
+
+### Prestate
+
+The alloc portion contains one contract (`0x000000000000000000000000000000000000aaaa`), containing the 
+following code: `0x5854505854`: `PC ;SLOAD; POP; PC; SLOAD`.
+
+Essentialy, this contract does `SLOAD(0)` and `SLOAD(3)`.
+
+The alloc also contains some funds on `0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b`. 
+
+## Transactions
+
+There are three transactions, each invokes the contract above. 
+
+1. ACL-transaction, which contains some non-used slots
+2. Regular transaction
+3. ACL-transaction, which contains the slots `1` and `3` in `0x000000000000000000000000000000000000aaaa`
+
+## Execution 
+
+Running it yields: 
+```
+dir=./testdata/8 && ./evm t8n --state.fork=Berlin --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --trace && cat trace-* | grep SLOAD
+{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":4,"op":84,"gas":"0x47c86","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":1,"op":84,"gas":"0x49cf6","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":4,"op":84,"gas":"0x494be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":4,"op":84,"gas":"0x48456","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+
+```
+
+Simlarly, we can provide the input transactions via `stdin` instead of as file: 
+
+```
+dir=./testdata/8 \
+  && cat $dir/txs.json | jq "{txs: .}" \
+  | ./evm t8n --state.fork=Berlin \
+     --input.alloc=$dir/alloc.json \
+     --input.txs=stdin \
+     --input.env=$dir/env.json \
+     --trace  \
+  && cat trace-* | grep SLOAD
+
+{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":4,"op":84,"gas":"0x47c86","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":1,"op":84,"gas":"0x49cf6","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":4,"op":84,"gas":"0x494be","gasCost":"0x834","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":1,"op":84,"gas":"0x484be","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x0"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+{"pc":4,"op":84,"gas":"0x48456","gasCost":"0x64","memory":"0x","memSize":0,"stack":["0x3"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"SLOAD","error":""}
+```
+
+If we try to execute it on older rules: 
+```
+dir=./testdata/8 && ./evm t8n --state.fork=Istanbul --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json 
+INFO [01-21|23:21:51.265] rejected tx                              index=0 hash="d2818d…6ab3da" error="tx type not supported"
+INFO [01-21|23:21:51.265] rejected tx                              index=1 hash="26ea00…81c01b" from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0"
+INFO [01-21|23:21:51.265] rejected tx                              index=2 hash="698d01…369cee" error="tx type not supported"
+```
+Number `1` and `3` are not applicable, and therefore number `2` has wrong nonce. 

cmd/evm/testdata/8/txs.json

@@ -0,0 +1,58 @@
+[
+  {
+    "gas": "0x4ef00",
+    "gasPrice": "0x1",
+    "chainId": "0x1",
+    "input": "0x",
+    "nonce": "0x0",
+    "to": "0x000000000000000000000000000000000000aaaa",
+    "value": "0x1",
+    "type" : "0x1",
+    "accessList": [
+      {"address": "0x0000000000000000000000000000000000000000",
+        "storageKeys": [
+          "0x0000000000000000000000000000000000000000000000000000000000000000",
+          "0x0000000000000000000000000000000000000000000000000000000000000000"
+        ]
+      }
+    ],
+    "v": "0x0",
+    "r": "0x0",
+    "s": "0x0",
+    "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"
+  },
+  {
+    "gas": "0x4ef00",
+    "gasPrice": "0x1",
+    "input": "0x",
+    "nonce": "0x1",
+    "to": "0x000000000000000000000000000000000000aaaa",
+    "value": "0x2",
+    "v": "0x0",
+    "r": "0x0",
+    "s": "0x0",
+    "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"
+  },
+  {
+    "gas": "0x4ef00",
+    "gasPrice": "0x1",
+    "chainId": "0x1",
+    "input": "0x",
+    "nonce": "0x2",
+    "to": "0x000000000000000000000000000000000000aaaa",
+    "value": "0x1",
+    "type" : "0x1",
+    "accessList": [
+      {"address": "0x000000000000000000000000000000000000aaaa",
+        "storageKeys": [
+          "0x0000000000000000000000000000000000000000000000000000000000000000",
+          "0x0000000000000000000000000000000000000000000000000000000000000003"
+        ]
+      }
+    ],
+    "v": "0x0",
+    "r": "0x0",
+    "s": "0x0",
+    "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"
+  }
+]

cmd/faucet/README.md

@@ -0,0 +1,50 @@
+# Faucet
+
+The `faucet` is a simplistic web application with the goal of distributing small amounts of Ether in private and test networks.
+
+Users need to post their Ethereum addresses to fund in a Twitter status update or public Facebook post and share the link to the faucet. The faucet will in turn deduplicate user requests and send the Ether. After a funding round, the faucet prevents the same user requesting again for a pre-configured amount of time, proportional to the amount of Ether requested.
+
+## Operation
+
+The `faucet` is a single binary app (everything included) with all configurations set via command line flags and a few files.
+
+First thing's first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set:
+
+- `--genesis` is a path to a file containin the network `genesis.json`
+- `--network` is the devp2p network id used during connection
+- `--bootnodes` is a list of `enode://` ids to join the network through
+
+The `faucet` will use the `les` protocol to join the configured Ethereum network and will store its data in `$HOME/.faucet` (currently not configurable).
+
+## Funding
+
+To be able to distribute funds, the `faucet` needs access to an already funded Ethereum account. This can be configured via:
+
+- `--account.json` is a path to the Ethereum account's JSON key file
+- `--account.pass` is a path to a text file with the decryption passphrase
+
+The faucet is able to distribute various amounts of Ether in exchange for various timeouts. These can be configured via:
+
+- `--faucet.amount` is the number of Ethers to send by default
+- `--faucet.minutes` is the time to wait before allowing a rerequest
+- `--faucet.tiers` is the funding tiers to support  (x3 time, x2.5 funds)
+
+## Sybil protection
+
+To prevent the same user from exhausting funds in a loop, the `faucet` ties requests to social networks and captcha resolvers.
+
+Captcha protection uses Google's invisible ReCaptcha, thus the `faucet` needs to run on a live domain. The domain needs to be registered in Google's systems to retrieve the captcha API token and secrets. After doing so, captcha protection may be enabled via:
+
+- `--captcha.token` is the API token for ReCaptcha
+- `--captcha.secret` is the API secret for ReCaptcha
+
+Sybil protection via Twitter requires an API key as of 15th December, 2020. To obtain it, a Twitter user must be upgraded to developer status and a new Twitter App deployed with it. The app's `Bearer` token is required by the faucet to retrieve tweet data:
+
+- `--twitter.token` is the Bearer token for `v2` API access
+- `--twitter.token.v1` is the Bearer token for `v1` API access
+
+Sybil protection via Facebook uses the website to directly download post data thus does not currently require an API configuration. 
+
+## Miscellaneous
+
+Beside the above - mostly essential - CLI flags, there are a number that can be used to fine tune the `faucet`'s operation. Please see `faucet --help` for a full list.

cmd/faucet/faucet.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// faucet is a Ether faucet backed by a light client.
+// faucet is an Ether faucet backed by a light client.
 package main
 
 //go:generate go-bindata -nometadata -o website.go faucet.html
@@ -47,15 +47,14 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/downloader"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/ethstats"
 	"github.com/ethereum/go-ethereum/les"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discv5"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
@@ -83,6 +82,9 @@ var (
 
 	noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication")
 	logFlag    = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+
+	twitterTokenFlag   = flag.String("twitter.token", "", "Bearer token to authenticate with the v2 Twitter API")
+	twitterTokenV1Flag = flag.String("twitter.token.v1", "", "Bearer token to authenticate with the v1.1 Twitter API")
 )
 
 var (
@@ -151,9 +153,9 @@ func main() {
 		log.Crit("Failed to parse genesis block json", "err", err)
 	}
 	// Convert the bootnodes to internal enode representations
-	var enodes []*discv5.Node
+	var enodes []*enode.Node
 	for _, boot := range strings.Split(*bootFlag, ",") {
-		if url, err := discv5.ParseNode(boot); err == nil {
+		if url, err := enode.Parse(enode.ValidSchemes, boot); err == nil {
 			enodes = append(enodes, url)
 		} else {
 			log.Error("Failed to parse bootnode URL", "url", boot, "err", err)
@@ -210,7 +212,7 @@ type faucet struct {
 	nonce    uint64             // Current pending nonce of the faucet
 	price    *big.Int           // Current gas price to issue funds with
 
-	conns    []*websocket.Conn    // Currently live websocket connections
+	conns    []*wsConn            // Currently live websocket connections
 	timeouts map[string]time.Time // History of users and their funding timeouts
 	reqs     []*request           // Currently pending funding requests
 	update   chan struct{}        // Channel to signal request updates
@@ -218,7 +220,14 @@ type faucet struct {
 	lock sync.RWMutex // Lock protecting the faucet's internals
 }
 
-func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) {
+// wsConn wraps a websocket connection with a write mutex as the underlying
+// websocket library does not synchronize access to the stream.
+type wsConn struct {
+	conn  *websocket.Conn
+	wlock sync.Mutex
+}
+
+func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) {
 	// Assemble the raw devp2p protocol stack
 	stack, err := node.New(&node.Config{
 		Name:    "geth",
@@ -238,11 +247,12 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 	}
 
 	// Assemble the Ethereum light client protocol
-	cfg := eth.DefaultConfig
+	cfg := ethconfig.Defaults
 	cfg.SyncMode = downloader.LightSync
 	cfg.NetworkId = network
 	cfg.Genesis = genesis
 	utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash())
+
 	lesBackend, err := les.New(stack, &cfg)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to register the Ethereum service: %w", err)
@@ -317,13 +327,14 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 	defer conn.Close()
 
 	f.lock.Lock()
-	f.conns = append(f.conns, conn)
+	wsconn := &wsConn{conn: conn}
+	f.conns = append(f.conns, wsconn)
 	f.lock.Unlock()
 
 	defer func() {
 		f.lock.Lock()
 		for i, c := range f.conns {
-			if c == conn {
+			if c.conn == conn {
 				f.conns = append(f.conns[:i], f.conns[i+1:]...)
 				break
 			}
@@ -351,7 +362,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		if head == nil || balance == nil {
 			// Report the faucet offline until initial stats are ready
 			//lint:ignore ST1005 This error is to be displayed in the browser
-			if err = sendError(conn, errors.New("Faucet offline")); err != nil {
+			if err = sendError(wsconn, errors.New("Faucet offline")); err != nil {
 				log.Warn("Failed to send faucet error to client", "err", err)
 				return
 			}
@@ -362,7 +373,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 	f.lock.RLock()
 	reqs := f.reqs
 	f.lock.RUnlock()
-	if err = send(conn, map[string]interface{}{
+	if err = send(wsconn, map[string]interface{}{
 		"funds":    new(big.Int).Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
@@ -371,7 +382,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		log.Warn("Failed to send initial stats to client", "err", err)
 		return
 	}
-	if err = send(conn, head, 3*time.Second); err != nil {
+	if err = send(wsconn, head, 3*time.Second); err != nil {
 		log.Warn("Failed to send initial header to client", "err", err)
 		return
 	}
@@ -386,9 +397,8 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		if err = conn.ReadJSON(&msg); err != nil {
 			return
 		}
-		if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://gist.github.com/") && !strings.HasPrefix(msg.URL, "https://twitter.com/") &&
-			!strings.HasPrefix(msg.URL, "https://plus.google.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") {
-			if err = sendError(conn, errors.New("URL doesn't link to supported services")); err != nil {
+		if !*noauthFlag && !strings.HasPrefix(msg.URL, "https://twitter.com/") && !strings.HasPrefix(msg.URL, "https://www.facebook.com/") {
+			if err = sendError(wsconn, errors.New("URL doesn't link to supported services")); err != nil {
 				log.Warn("Failed to send URL error to client", "err", err)
 				return
 			}
@@ -396,7 +406,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		}
 		if msg.Tier >= uint(*tiersFlag) {
 			//lint:ignore ST1005 This error is to be displayed in the browser
-			if err = sendError(conn, errors.New("Invalid funding tier requested")); err != nil {
+			if err = sendError(wsconn, errors.New("Invalid funding tier requested")); err != nil {
 				log.Warn("Failed to send tier error to client", "err", err)
 				return
 			}
@@ -412,7 +422,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 
 			res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form)
 			if err != nil {
-				if err = sendError(conn, err); err != nil {
+				if err = sendError(wsconn, err); err != nil {
 					log.Warn("Failed to send captcha post error to client", "err", err)
 					return
 				}
@@ -425,7 +435,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			err = json.NewDecoder(res.Body).Decode(&result)
 			res.Body.Close()
 			if err != nil {
-				if err = sendError(conn, err); err != nil {
+				if err = sendError(wsconn, err); err != nil {
 					log.Warn("Failed to send captcha decode error to client", "err", err)
 					return
 				}
@@ -434,7 +444,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			if !result.Success {
 				log.Warn("Captcha verification failed", "err", string(result.Errors))
 				//lint:ignore ST1005 it's funny and the robot won't mind
-				if err = sendError(conn, errors.New("Beep-bop, you're a robot!")); err != nil {
+				if err = sendError(wsconn, errors.New("Beep-bop, you're a robot!")); err != nil {
 					log.Warn("Failed to send captcha failure to client", "err", err)
 					return
 				}
@@ -443,36 +453,26 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		}
 		// Retrieve the Ethereum address to fund, the requesting user and a profile picture
 		var (
+			id       string
 			username string
 			avatar   string
 			address  common.Address
 		)
 		switch {
-		case strings.HasPrefix(msg.URL, "https://gist.github.com/"):
-			if err = sendError(conn, errors.New("GitHub authentication discontinued at the official request of GitHub")); err != nil {
-				log.Warn("Failed to send GitHub deprecation to client", "err", err)
-				return
-			}
-			continue
-		case strings.HasPrefix(msg.URL, "https://plus.google.com/"):
-			//lint:ignore ST1005 Google is a company name and should be capitalized.
-			if err = sendError(conn, errors.New("Google+ authentication discontinued as the service was sunset")); err != nil {
-				log.Warn("Failed to send Google+ deprecation to client", "err", err)
-				return
-			}
-			continue
 		case strings.HasPrefix(msg.URL, "https://twitter.com/"):
-			username, avatar, address, err = authTwitter(msg.URL)
+			id, username, avatar, address, err = authTwitter(msg.URL, *twitterTokenV1Flag, *twitterTokenFlag)
 		case strings.HasPrefix(msg.URL, "https://www.facebook.com/"):
 			username, avatar, address, err = authFacebook(msg.URL)
+			id = username
 		case *noauthFlag:
 			username, avatar, address, err = authNoAuth(msg.URL)
+			id = username
 		default:
 			//lint:ignore ST1005 This error is to be displayed in the browser
 			err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues")
 		}
 		if err != nil {
-			if err = sendError(conn, err); err != nil {
+			if err = sendError(wsconn, err); err != nil {
 				log.Warn("Failed to send prefix error to client", "err", err)
 				return
 			}
@@ -486,7 +486,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			fund    bool
 			timeout time.Time
 		)
-		if timeout = f.timeouts[username]; time.Now().After(timeout) {
+		if timeout = f.timeouts[id]; time.Now().After(timeout) {
 			// User wasn't funded recently, create the funding transaction
 			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
 			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
@@ -496,7 +496,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainID)
 			if err != nil {
 				f.lock.Unlock()
-				if err = sendError(conn, err); err != nil {
+				if err = sendError(wsconn, err); err != nil {
 					log.Warn("Failed to send transaction creation error to client", "err", err)
 					return
 				}
@@ -505,7 +505,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			// Submit the transaction and mark as funded if successful
 			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
 				f.lock.Unlock()
-				if err = sendError(conn, err); err != nil {
+				if err = sendError(wsconn, err); err != nil {
 					log.Warn("Failed to send transaction transmission error to client", "err", err)
 					return
 				}
@@ -520,20 +520,20 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute
 			grace := timeout / 288 // 24h timeout => 5m grace
 
-			f.timeouts[username] = time.Now().Add(timeout - grace)
+			f.timeouts[id] = time.Now().Add(timeout - grace)
 			fund = true
 		}
 		f.lock.Unlock()
 
 		// Send an error if too frequent funding, othewise a success
 		if !fund {
-			if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(time.Until(timeout)))); err != nil { // nolint: gosimple
+			if err = sendError(wsconn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(time.Until(timeout)))); err != nil { // nolint: gosimple
 				log.Warn("Failed to send funding error to client", "err", err)
 				return
 			}
 			continue
 		}
-		if err = sendSuccess(conn, fmt.Sprintf("Funding request accepted for %s into %s", username, address.Hex())); err != nil {
+		if err = sendSuccess(wsconn, fmt.Sprintf("Funding request accepted for %s into %s", username, address.Hex())); err != nil {
 			log.Warn("Failed to send funding success to client", "err", err)
 			return
 		}
@@ -626,12 +626,12 @@ func (f *faucet) loop() {
 					"requests": f.reqs,
 				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)
-					conn.Close()
+					conn.conn.Close()
 					continue
 				}
 				if err := send(conn, head, time.Second); err != nil {
 					log.Warn("Failed to send header to client", "err", err)
-					conn.Close()
+					conn.conn.Close()
 				}
 			}
 			f.lock.RUnlock()
@@ -653,7 +653,7 @@ func (f *faucet) loop() {
 			for _, conn := range f.conns {
 				if err := send(conn, map[string]interface{}{"requests": f.reqs}, time.Second); err != nil {
 					log.Warn("Failed to send requests to client", "err", err)
-					conn.Close()
+					conn.conn.Close()
 				}
 			}
 			f.lock.RUnlock()
@@ -663,44 +663,63 @@ func (f *faucet) loop() {
 
 // sends transmits a data packet to the remote end of the websocket, but also
 // setting a write deadline to prevent waiting forever on the node.
-func send(conn *websocket.Conn, value interface{}, timeout time.Duration) error {
+func send(conn *wsConn, value interface{}, timeout time.Duration) error {
 	if timeout == 0 {
 		timeout = 60 * time.Second
 	}
-	conn.SetWriteDeadline(time.Now().Add(timeout))
-	return conn.WriteJSON(value)
+	conn.wlock.Lock()
+	defer conn.wlock.Unlock()
+	conn.conn.SetWriteDeadline(time.Now().Add(timeout))
+	return conn.conn.WriteJSON(value)
 }
 
 // sendError transmits an error to the remote end of the websocket, also setting
 // the write deadline to 1 second to prevent waiting forever.
-func sendError(conn *websocket.Conn, err error) error {
+func sendError(conn *wsConn, err error) error {
 	return send(conn, map[string]string{"error": err.Error()}, time.Second)
 }
 
 // sendSuccess transmits a success message to the remote end of the websocket, also
 // setting the write deadline to 1 second to prevent waiting forever.
-func sendSuccess(conn *websocket.Conn, msg string) error {
+func sendSuccess(conn *wsConn, msg string) error {
 	return send(conn, map[string]string{"success": msg}, time.Second)
 }
 
 // authTwitter tries to authenticate a faucet request using Twitter posts, returning
-// the username, avatar URL and Ethereum address to fund on success.
-func authTwitter(url string) (string, string, common.Address, error) {
+// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success.
+func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, common.Address, error) {
 	// Ensure the user specified a meaningful URL, no fancy nonsense
 	parts := strings.Split(url, "/")
 	if len(parts) < 4 || parts[len(parts)-2] != "status" {
 		//lint:ignore ST1005 This error is to be displayed in the browser
-		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+		return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL")
 	}
-	// Twitter's API isn't really friendly with direct links. Still, we don't
-	// want to do ask read permissions from users, so just load the public posts
+	// Strip any query parameters from the tweet id and ensure it's numeric
+	tweetID := strings.Split(parts[len(parts)-1], "?")[0]
+	if !regexp.MustCompile("^[0-9]+$").MatchString(tweetID) {
+		return "", "", "", common.Address{}, errors.New("Invalid Tweet URL")
+	}
+	// Twitter's API isn't really friendly with direct links.
+	// It is restricted to 300 queries / 15 minute with an app api key.
+	// Anything more will require read only authorization from the users and that we want to avoid.
+
+	// If Twitter bearer token is provided, use the API, selecting the version
+	// the user would prefer (currently there's a limit of 1 v2 app / developer
+	// but unlimited v1.1 apps).
+	switch {
+	case tokenV1 != "":
+		return authTwitterWithTokenV1(tweetID, tokenV1)
+	case tokenV2 != "":
+		return authTwitterWithTokenV2(tweetID, tokenV2)
+	}
+	// Twiter API token isn't provided so we just load the public posts
 	// and scrape it for the Ethereum address and profile URL. We need to load
 	// the mobile page though since the main page loads tweet contents via JS.
 	url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1)
 
 	res, err := http.Get(url)
 	if err != nil {
-		return "", "", common.Address{}, err
+		return "", "", "", common.Address{}, err
 	}
 	defer res.Body.Close()
 
@@ -708,31 +727,115 @@ func authTwitter(url string) (string, string, common.Address, error) {
 	parts = strings.Split(res.Request.URL.String(), "/")
 	if len(parts) < 4 || parts[len(parts)-2] != "status" {
 		//lint:ignore ST1005 This error is to be displayed in the browser
-		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+		return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL")
 	}
 	username := parts[len(parts)-3]
 
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return "", "", common.Address{}, err
+		return "", "", "", common.Address{}, err
 	}
 	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
 	if address == (common.Address{}) {
 		//lint:ignore ST1005 This error is to be displayed in the browser
-		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
 	}
 	var avatar string
 	if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
 		avatar = parts[1]
 	}
-	return username + "@twitter", avatar, address, nil
+	return username + "@twitter", username, avatar, address, nil
+}
+
+// authTwitterWithTokenV1 tries to authenticate a faucet request using Twitter's v1
+// API, returning the user id, username, avatar URL and Ethereum address to fund on
+// success.
+func authTwitterWithTokenV1(tweetID string, token string) (string, string, string, common.Address, error) {
+	// Query the tweet details from Twitter
+	url := fmt.Sprintf("https://api.twitter.com/1.1/statuses/show.json?id=%s", tweetID)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	var result struct {
+		Text string `json:"text"`
+		User struct {
+			ID       string `json:"id_str"`
+			Username string `json:"screen_name"`
+			Avatar   string `json:"profile_image_url"`
+		} `json:"user"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&result)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Text))
+	if address == (common.Address{}) {
+		//lint:ignore ST1005 This error is to be displayed in the browser
+		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	return result.User.ID + "@twitter", result.User.Username, result.User.Avatar, address, nil
+}
+
+// authTwitterWithTokenV2 tries to authenticate a faucet request using Twitter's v2
+// API, returning the user id, username, avatar URL and Ethereum address to fund on
+// success.
+func authTwitterWithTokenV2(tweetID string, token string) (string, string, string, common.Address, error) {
+	// Query the tweet details from Twitter
+	url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", tweetID)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	var result struct {
+		Data struct {
+			AuthorID string `json:"author_id"`
+			Text     string `json:"text"`
+		} `json:"data"`
+		Includes struct {
+			Users []struct {
+				ID       string `json:"id"`
+				Username string `json:"username"`
+				Avatar   string `json:"profile_image_url"`
+			} `json:"users"`
+		} `json:"includes"`
+	}
+
+	err = json.NewDecoder(res.Body).Decode(&result)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+
+	address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text))
+	if address == (common.Address{}) {
+		//lint:ignore ST1005 This error is to be displayed in the browser
+		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].Avatar, address, nil
 }
 
 // authFacebook tries to authenticate a faucet request using Facebook posts,
 // returning the username, avatar URL and Ethereum address to fund on success.
 func authFacebook(url string) (string, string, common.Address, error) {
 	// Ensure the user specified a meaningful URL, no fancy nonsense
-	parts := strings.Split(url, "/")
+	parts := strings.Split(strings.Split(url, "?")[0], "/")
+	if parts[len(parts)-1] == "" {
+		parts = parts[0 : len(parts)-1]
+	}
 	if len(parts) < 4 || parts[len(parts)-2] != "posts" {
 		//lint:ignore ST1005 This error is to be displayed in the browser
 		return "", "", common.Address{}, errors.New("Invalid Facebook post URL")
@@ -742,7 +845,13 @@ func authFacebook(url string) (string, string, common.Address, error) {
 	// Facebook's Graph API isn't really friendly with direct links. Still, we don't
 	// want to do ask read permissions from users, so just load the public posts and
 	// scrape it for the Ethereum address and profile URL.
-	res, err := http.Get(url)
+	//
+	// Facebook recently changed their desktop webpage to use AJAX for loading post
+	// content, so switch over to the mobile site for now. Will probably end up having
+	// to use the API eventually.
+	crawl := strings.Replace(url, "www.facebook.com", "m.facebook.com", 1)
+
+	res, err := http.Get(crawl)
 	if err != nil {
 		return "", "", common.Address{}, err
 	}

cmd/faucet/faucet.html

@@ -177,7 +177,7 @@
 						}
 						// Iterate over our entire local collection and re-render the funding table
 						var content = "";
-						for (var i=0; i<requests.length; i++) {
+						for (var i=requests.length-1; i >= 0; i--) {
 							var done    = requests[i].time == "";
 							var elapsed = moment().unix()-moment(requests[i].time).unix();
 

cmd/faucet/faucet_test.go

@@ -0,0 +1,43 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+func TestFacebook(t *testing.T) {
+	for _, tt := range []struct {
+		url  string
+		want common.Address
+	}{
+		{
+			"https://www.facebook.com/fooz.gazonk/posts/2837228539847129",
+			common.HexToAddress("0xDeadDeaDDeaDbEefbEeFbEEfBeeFBeefBeeFbEEF"),
+		},
+	} {
+		_, _, gotAddress, err := authFacebook(tt.url)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if gotAddress != tt.want {
+			t.Fatalf("address wrong, have %v want %v", gotAddress, tt.want)
+		}
+	}
+}

cmd/geth/accountcmd_test.go

@@ -102,6 +102,7 @@ func TestAccountImport(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
+		test := test
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 			importAccountWithExpect(t, test.key, test.output)
@@ -178,11 +179,8 @@ Fatal: could not decrypt key with given password
 }
 
 func TestUnlockFlag(t *testing.T) {
-	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
-		"js", "testdata/empty.js")
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "js", "testdata/empty.js")
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
@@ -202,10 +200,9 @@ Password: {{.InputLine "foobar"}}
 }
 
 func TestUnlockFlagWrongPassword(t *testing.T) {
-	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "js", "testdata/empty.js")
+
 	defer geth.ExpectExit()
 	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
@@ -221,11 +218,9 @@ Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could
 
 // https://github.com/ethereum/go-ethereum/issues/1785
 func TestUnlockFlagMultiIndex(t *testing.T) {
-	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--unlock", "0,2",
-		"js", "testdata/empty.js")
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--unlock", "0,2", "js", "testdata/empty.js")
+
 	geth.Expect(`
 Unlocking account 0 | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
@@ -248,11 +243,9 @@ Password: {{.InputLine "foobar"}}
 }
 
 func TestUnlockFlagPasswordFile(t *testing.T) {
-	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--password", "testdata/passwords.txt", "--unlock", "0,2",
-		"js", "testdata/empty.js")
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--password", "testdata/passwords.txt", "--unlock", "0,2", "js", "testdata/empty.js")
+
 	geth.ExpectExit()
 
 	wantMessages := []string{
@@ -268,10 +261,9 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 }
 
 func TestUnlockFlagPasswordFileWrongPassword(t *testing.T) {
-	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
-		"--datadir", datadir, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--password",
+		"testdata/wrong-passwords.txt", "--unlock", "0,2")
 	defer geth.ExpectExit()
 	geth.Expect(`
 Fatal: Failed to unlock account 0 (could not decrypt key with given password)
@@ -280,9 +272,9 @@ Fatal: Failed to unlock account 0 (could not decrypt key with given password)
 
 func TestUnlockFlagAmbiguous(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
-	geth := runGeth(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--keystore",
+		store, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
 	defer geth.ExpectExit()
 
@@ -318,9 +310,10 @@ In order to avoid this warning, you need to remove the following duplicate key f
 
 func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
 	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
-	geth := runGeth(t,
-		"--keystore", store, "--nat", "none", "--nodiscover", "--maxpeers", "0", "--port", "0",
-		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
+	geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t),
+		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--keystore",
+		store, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
+
 	defer geth.ExpectExit()
 
 	// Helper for the expect template, returns absolute keystore path.

cmd/geth/chaincmd.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strconv"
 	"sync/atomic"
@@ -28,7 +27,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -159,28 +157,16 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.FakePoWFlag,
+			utils.MainnetFlag,
 			utils.RopstenFlag,
 			utils.RinkebyFlag,
 			utils.TxLookupLimitFlag,
 			utils.GoerliFlag,
-			utils.YoloV1Flag,
-			utils.LegacyTestnetFlag,
+			utils.YoloV3Flag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The first argument must be the directory containing the blockchain to download from`,
-	}
-	removedbCommand = cli.Command{
-		Action:    utils.MigrateFlags(removeDB),
-		Name:      "removedb",
-		Usage:     "Remove blockchain and state databases",
-		ArgsUsage: " ",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-		},
-		Category: "BLOCKCHAIN COMMANDS",
-		Description: `
-Remove blockchain and state databases`,
 	}
 	dumpCommand = cli.Command{
 		Action:    utils.MigrateFlags(dump),
@@ -201,24 +187,6 @@ Remove blockchain and state databases`,
 The arguments are interpreted as block numbers or hashes.
 Use "ethereum dump 0" to dump the genesis block.`,
 	}
-	inspectCommand = cli.Command{
-		Action:    utils.MigrateFlags(inspect),
-		Name:      "inspect",
-		Usage:     "Inspect the storage size for each type of data in the database",
-		ArgsUsage: " ",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-			utils.AncientFlag,
-			utils.CacheFlag,
-			utils.RopstenFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-			utils.YoloV1Flag,
-			utils.LegacyTestnetFlag,
-			utils.SyncModeFlag,
-		},
-		Category: "BLOCKCHAIN COMMANDS",
-	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -321,17 +289,7 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Import done in %v.\n\n", time.Since(start))
 
 	// Output pre-compaction stats mostly to see the import trashing
-	stats, err := db.Stat("leveldb.stats")
-	if err != nil {
-		utils.Fatalf("Failed to read database stats: %v", err)
-	}
-	fmt.Println(stats)
-
-	ioStats, err := db.Stat("leveldb.iostats")
-	if err != nil {
-		utils.Fatalf("Failed to read database iostats: %v", err)
-	}
-	fmt.Println(ioStats)
+	showLeveldbStats(db)
 
 	// Print the memory statistics used by the importing
 	mem := new(runtime.MemStats)
@@ -349,22 +307,12 @@ func importChain(ctx *cli.Context) error {
 	// Compact the entire database to more accurately measure disk io and print the stats
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
-	if err = db.Compact(nil, nil); err != nil {
+	if err := db.Compact(nil, nil); err != nil {
 		utils.Fatalf("Compaction failed: %v", err)
 	}
 	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
 
-	stats, err = db.Stat("leveldb.stats")
-	if err != nil {
-		utils.Fatalf("Failed to read database stats: %v", err)
-	}
-	fmt.Println(stats)
-
-	ioStats, err = db.Stat("leveldb.iostats")
-	if err != nil {
-		utils.Fatalf("Failed to read database iostats: %v", err)
-	}
-	fmt.Println(ioStats)
+	showLeveldbStats(db)
 	return importErr
 }
 
@@ -497,66 +445,6 @@ func copyDb(ctx *cli.Context) error {
 	return nil
 }
 
-func removeDB(ctx *cli.Context) error {
-	stack, config := makeConfigNode(ctx)
-
-	// Remove the full node state database
-	path := stack.ResolvePath("chaindata")
-	if common.FileExist(path) {
-		confirmAndRemoveDB(path, "full node state database")
-	} else {
-		log.Info("Full node state database missing", "path", path)
-	}
-	// Remove the full node ancient database
-	path = config.Eth.DatabaseFreezer
-	switch {
-	case path == "":
-		path = filepath.Join(stack.ResolvePath("chaindata"), "ancient")
-	case !filepath.IsAbs(path):
-		path = config.Node.ResolvePath(path)
-	}
-	if common.FileExist(path) {
-		confirmAndRemoveDB(path, "full node ancient database")
-	} else {
-		log.Info("Full node ancient database missing", "path", path)
-	}
-	// Remove the light node database
-	path = stack.ResolvePath("lightchaindata")
-	if common.FileExist(path) {
-		confirmAndRemoveDB(path, "light node database")
-	} else {
-		log.Info("Light node database missing", "path", path)
-	}
-	return nil
-}
-
-// confirmAndRemoveDB prompts the user for a last confirmation and removes the
-// folder if accepted.
-func confirmAndRemoveDB(database string, kind string) {
-	confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database))
-	switch {
-	case err != nil:
-		utils.Fatalf("%v", err)
-	case !confirm:
-		log.Info("Database deletion skipped", "path", database)
-	default:
-		start := time.Now()
-		filepath.Walk(database, func(path string, info os.FileInfo, err error) error {
-			// If we're at the top level folder, recurse into
-			if path == database {
-				return nil
-			}
-			// Delete all the files, but not subfolders
-			if !info.IsDir() {
-				os.Remove(path)
-				return nil
-			}
-			return filepath.SkipDir
-		})
-		log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start)))
-	}
-}
-
 func dump(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
@@ -596,16 +484,6 @@ func dump(ctx *cli.Context) error {
 	return nil
 }
 
-func inspect(ctx *cli.Context) error {
-	node, _ := makeConfigNode(ctx)
-	defer node.Close()
-
-	_, chainDb := utils.MakeChain(ctx, node, true)
-	defer chainDb.Close()
-
-	return rawdb.InspectDatabase(chainDb)
-}
-
 // hashish returns true for strings that look like hashes.
 func hashish(x string) bool {
 	_, err := strconv.Atoi(x)

cmd/geth/config.go

@@ -20,16 +20,17 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"math/big"
 	"os"
 	"reflect"
 	"unicode"
 
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/naoina/toml"
@@ -41,7 +42,7 @@ var (
 		Name:        "dumpconfig",
 		Usage:       "Show configuration values",
 		ArgsUsage:   "",
-		Flags:       append(append(nodeFlags, rpcFlags...), whisperFlags...),
+		Flags:       append(nodeFlags, rpcFlags...),
 		Category:    "MISCELLANEOUS COMMANDS",
 		Description: `The dumpconfig command shows configuration values.`,
 	}
@@ -73,21 +74,11 @@ type ethstatsConfig struct {
 	URL string `toml:",omitempty"`
 }
 
-// whisper has been deprecated, but clients out there might still have [Shh]
-// in their config, which will crash. Cut them some slack by keeping the
-// config, and displaying a message that those config switches are ineffectual.
-// To be removed circa Q1 2021 -- @gballet.
-type whisperDeprecatedConfig struct {
-	MaxMessageSize                        uint32  `toml:",omitempty"`
-	MinimumAcceptedPOW                    float64 `toml:",omitempty"`
-	RestrictConnectionBetweenLightClients bool    `toml:",omitempty"`
-}
-
 type gethConfig struct {
-	Eth      eth.Config
-	Shh      whisperDeprecatedConfig
+	Eth      ethconfig.Config
 	Node     node.Config
 	Ethstats ethstatsConfig
+	Metrics  metrics.Config
 }
 
 func loadConfig(file string, cfg *gethConfig) error {
@@ -119,8 +110,9 @@ func defaultNodeConfig() node.Config {
 func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	// Load defaults.
 	cfg := gethConfig{
-		Eth:  eth.DefaultConfig,
-		Node: defaultNodeConfig(),
+		Eth:     ethconfig.Defaults,
+		Node:    defaultNodeConfig(),
+		Metrics: metrics.DefaultConfig,
 	}
 
 	// Load config file.
@@ -128,10 +120,6 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 		if err := loadConfig(file, &cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
-
-		if cfg.Shh != (whisperDeprecatedConfig{}) {
-			log.Warn("Deprecated whisper config detected. Whisper has been moved to github.com/ethereum/whisper")
-		}
 	}
 
 	// Apply flags.
@@ -144,27 +132,19 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) {
 		cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
 	}
-	utils.SetShhConfig(ctx, stack)
+	applyMetricConfig(ctx, &cfg)
 
 	return stack, cfg
 }
 
-// enableWhisper returns true in case one of the whisper flags is set.
-func checkWhisper(ctx *cli.Context) {
-	for _, flag := range whisperFlags {
-		if ctx.GlobalIsSet(flag.GetName()) {
-			log.Warn("deprecated whisper flag detected. Whisper has been moved to github.com/ethereum/whisper")
-		}
-	}
-}
-
 // makeFullNode loads geth configuration and creates the Ethereum backend.
 func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	stack, cfg := makeConfigNode(ctx)
-
+	if ctx.GlobalIsSet(utils.OverrideBerlinFlag.Name) {
+		cfg.Eth.OverrideBerlin = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideBerlinFlag.Name))
+	}
 	backend := utils.RegisterEthService(stack, &cfg.Eth)
 
-	checkWhisper(ctx)
 	// Configure GraphQL if requested
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {
 		utils.RegisterGraphQLService(stack, backend, cfg.Node)
@@ -204,3 +184,36 @@ func dumpConfig(ctx *cli.Context) error {
 
 	return nil
 }
+
+func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) {
+	if ctx.GlobalIsSet(utils.MetricsEnabledFlag.Name) {
+		cfg.Metrics.Enabled = ctx.GlobalBool(utils.MetricsEnabledFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsEnabledExpensiveFlag.Name) {
+		cfg.Metrics.EnabledExpensive = ctx.GlobalBool(utils.MetricsEnabledExpensiveFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsHTTPFlag.Name) {
+		cfg.Metrics.HTTP = ctx.GlobalString(utils.MetricsHTTPFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsPortFlag.Name) {
+		cfg.Metrics.Port = ctx.GlobalInt(utils.MetricsPortFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBFlag.Name) {
+		cfg.Metrics.EnableInfluxDB = ctx.GlobalBool(utils.MetricsEnableInfluxDBFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsInfluxDBEndpointFlag.Name) {
+		cfg.Metrics.InfluxDBEndpoint = ctx.GlobalString(utils.MetricsInfluxDBEndpointFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsInfluxDBDatabaseFlag.Name) {
+		cfg.Metrics.InfluxDBDatabase = ctx.GlobalString(utils.MetricsInfluxDBDatabaseFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsInfluxDBUsernameFlag.Name) {
+		cfg.Metrics.InfluxDBUsername = ctx.GlobalString(utils.MetricsInfluxDBUsernameFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsInfluxDBPasswordFlag.Name) {
+		cfg.Metrics.InfluxDBPassword = ctx.GlobalString(utils.MetricsInfluxDBPasswordFlag.Name)
+	}
+	if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) {
+		cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name)
+	}
+}

cmd/geth/consolecmd.go

@@ -19,10 +19,8 @@ package main
 import (
 	"fmt"
 	"os"
-	"os/signal"
 	"path/filepath"
 	"strings"
-	"syscall"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/console"
@@ -38,12 +36,12 @@ var (
 		Action:   utils.MigrateFlags(localConsole),
 		Name:     "console",
 		Usage:    "Start an interactive JavaScript environment",
-		Flags:    append(append(append(nodeFlags, rpcFlags...), consoleFlags...), whisperFlags...),
+		Flags:    append(append(nodeFlags, rpcFlags...), consoleFlags...),
 		Category: "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`,
+See https://geth.ethereum.org/docs/interface/javascript-console.`,
 	}
 
 	attachCommand = cli.Command{
@@ -56,7 +54,7 @@ See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`,
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.
+See https://geth.ethereum.org/docs/interface/javascript-console.
 This command allows to open a console on a running geth node.`,
 	}
 
@@ -69,7 +67,7 @@ This command allows to open a console on a running geth node.`,
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The JavaScript VM exposes a node admin interface as well as the Ðapp
-JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console`,
+JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console`,
 	}
 )
 
@@ -123,7 +121,7 @@ func remoteConsole(ctx *cli.Context) error {
 			path = ctx.GlobalString(utils.DataDirFlag.Name)
 		}
 		if path != "" {
-			if ctx.GlobalBool(utils.LegacyTestnetFlag.Name) || ctx.GlobalBool(utils.RopstenFlag.Name) {
+			if ctx.GlobalBool(utils.RopstenFlag.Name) {
 				// Maintain compatibility with older Geth configurations storing the
 				// Ropsten database in `testnet` instead of `ropsten`.
 				legacyPath := filepath.Join(path, "testnet")
@@ -136,8 +134,8 @@ func remoteConsole(ctx *cli.Context) error {
 				path = filepath.Join(path, "rinkeby")
 			} else if ctx.GlobalBool(utils.GoerliFlag.Name) {
 				path = filepath.Join(path, "goerli")
-			} else if ctx.GlobalBool(utils.YoloV1Flag.Name) {
-				path = filepath.Join(path, "yolo-v1")
+			} else if ctx.GlobalBool(utils.YoloV3Flag.Name) {
+				path = filepath.Join(path, "yolo-v3")
 			}
 		}
 		endpoint = fmt.Sprintf("%s/geth.ipc", path)
@@ -218,13 +216,10 @@ func ephemeralConsole(ctx *cli.Context) error {
 			utils.Fatalf("Failed to execute %s: %v", file, err)
 		}
 	}
-	// Wait for pending callbacks, but stop for Ctrl-C.
-	abort := make(chan os.Signal, 1)
-	signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM)
 
 	go func() {
-		<-abort
-		os.Exit(0)
+		stack.Wait()
+		console.Stop(false)
 	}()
 	console.Stop(true)
 

cmd/geth/consolecmd_test.go

@@ -35,16 +35,25 @@ const (
 	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
+// spawns geth with the given command line args, using a set of flags to minimise
+// memory and disk IO. If the args don't set --datadir, the
+// child g gets a temporary data directory.
+func runMinimalGeth(t *testing.T, args ...string) *testgeth {
+	// --ropsten to make the 'writing genesis to disk' faster (no accounts)
+	// --networkid=1337 to avoid cache bump
+	// --syncmode=full to avoid allocating fast sync bloom
+	allArgs := []string{"--ropsten", "--networkid", "1337", "--syncmode=full", "--port", "0",
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64"}
+	return runGeth(t, append(allArgs, args...)...)
+}
+
 // Tests that a node embedded within a console can be started up properly and
 // then terminated by closing the input stream.
 func TestConsoleWelcome(t *testing.T) {
 	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
 
 	// Start a geth console, make sure it's cleaned up and terminate the console
-	geth := runGeth(t,
-		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
-		"--etherbase", coinbase,
-		"console")
+	geth := runMinimalGeth(t, "--miner.etherbase", coinbase, "console")
 
 	// Gather all the infos the welcome message needs to contain
 	geth.SetTemplateFunc("goos", func() string { return runtime.GOOS })
@@ -66,16 +75,20 @@ at block: 0 ({{niltime}})
  datadir: {{.Datadir}}
  modules: {{apis}}
 
+To exit, press ctrl-d
 > {{.InputLine "exit"}}
 `)
 	geth.ExpectExit()
 }
 
 // Tests that a console can be attached to a running node via various means.
-func TestIPCAttachWelcome(t *testing.T) {
+func TestAttachWelcome(t *testing.T) {
+	var (
+		ipc      string
+		httpPort string
+		wsPort   string
+	)
 	// Configure the instance for IPC attachment
-	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
-	var ipc string
 	if runtime.GOOS == "windows" {
 		ipc = `\\.\pipe\geth` + strconv.Itoa(trulyRandInt(100000, 999999))
 	} else {
@@ -83,51 +96,28 @@ func TestIPCAttachWelcome(t *testing.T) {
 		defer os.RemoveAll(ws)
 		ipc = filepath.Join(ws, "geth.ipc")
 	}
-	geth := runGeth(t,
-		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
-		"--etherbase", coinbase, "--ipcpath", ipc)
-
-	defer func() {
-		geth.Interrupt()
-		geth.ExpectExit()
-	}()
-
-	waitForEndpoint(t, ipc, 3*time.Second)
-	testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs)
-
-}
-
-func TestHTTPAttachWelcome(t *testing.T) {
-	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
-	port := strconv.Itoa(trulyRandInt(1024, 65536)) // Yeah, sometimes this will fail, sorry :P
-	geth := runGeth(t,
-		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
-		"--etherbase", coinbase, "--http", "--http.port", port)
-	defer func() {
-		geth.Interrupt()
-		geth.ExpectExit()
-	}()
-
-	endpoint := "http://127.0.0.1:" + port
-	waitForEndpoint(t, endpoint, 3*time.Second)
-	testAttachWelcome(t, geth, endpoint, httpAPIs)
-}
-
-func TestWSAttachWelcome(t *testing.T) {
-	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
-	port := strconv.Itoa(trulyRandInt(1024, 65536)) // Yeah, sometimes this will fail, sorry :P
-
-	geth := runGeth(t,
-		"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
-		"--etherbase", coinbase, "--ws", "--ws.port", port)
-	defer func() {
-		geth.Interrupt()
-		geth.ExpectExit()
-	}()
-
-	endpoint := "ws://127.0.0.1:" + port
-	waitForEndpoint(t, endpoint, 3*time.Second)
-	testAttachWelcome(t, geth, endpoint, httpAPIs)
+	// And HTTP + WS attachment
+	p := trulyRandInt(1024, 65533) // Yeah, sometimes this will fail, sorry :P
+	httpPort = strconv.Itoa(p)
+	wsPort = strconv.Itoa(p + 1)
+	geth := runMinimalGeth(t, "--miner.etherbase", "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182",
+		"--ipcpath", ipc,
+		"--http", "--http.port", httpPort,
+		"--ws", "--ws.port", wsPort)
+	t.Run("ipc", func(t *testing.T) {
+		waitForEndpoint(t, ipc, 3*time.Second)
+		testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs)
+	})
+	t.Run("http", func(t *testing.T) {
+		endpoint := "http://127.0.0.1:" + httpPort
+		waitForEndpoint(t, endpoint, 3*time.Second)
+		testAttachWelcome(t, geth, endpoint, httpAPIs)
+	})
+	t.Run("ws", func(t *testing.T) {
+		endpoint := "ws://127.0.0.1:" + wsPort
+		waitForEndpoint(t, endpoint, 3*time.Second)
+		testAttachWelcome(t, geth, endpoint, httpAPIs)
+	})
 }
 
 func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
@@ -159,6 +149,7 @@ at block: 0 ({{niltime}}){{if ipc}}
  datadir: {{datadir}}{{end}}
  modules: {{apis}}
 
+To exit, press ctrl-d
 > {{.InputLine "exit" }}
 `)
 	attach.ExpectExit()

cmd/geth/dao_test.go

@@ -115,10 +115,10 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
 		if err := ioutil.WriteFile(json, []byte(genesis), 0600); err != nil {
 			t.Fatalf("test %d: failed to write genesis file: %v", test, err)
 		}
-		runGeth(t, "--datadir", datadir, "init", json).WaitExit()
+		runGeth(t, "--datadir", datadir, "--networkid", "1337", "init", json).WaitExit()
 	} else {
 		// Force chain initialization
-		args := []string{"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
+		args := []string{"--port", "0", "--networkid", "1337", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
 		runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...).WaitExit()
 	}
 	// Retrieve the DAO config flag from the database

cmd/geth/dbcmd.go

@@ -0,0 +1,341 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/console/prompt"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/leveldb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var (
+	removedbCommand = cli.Command{
+		Action:    utils.MigrateFlags(removeDB),
+		Name:      "removedb",
+		Usage:     "Remove blockchain and state databases",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+		},
+		Category: "DATABASE COMMANDS",
+		Description: `
+Remove blockchain and state databases`,
+	}
+	dbCommand = cli.Command{
+		Name:      "db",
+		Usage:     "Low level database operations",
+		ArgsUsage: "",
+		Category:  "DATABASE COMMANDS",
+		Subcommands: []cli.Command{
+			dbInspectCmd,
+			dbStatCmd,
+			dbCompactCmd,
+			dbGetCmd,
+			dbDeleteCmd,
+			dbPutCmd,
+		},
+	}
+	dbInspectCmd = cli.Command{
+		Action:    utils.MigrateFlags(inspect),
+		Name:      "inspect",
+		ArgsUsage: "<prefix> <start>",
+
+		Usage:       "Inspect the storage size for each type of data in the database",
+		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
+	}
+	dbStatCmd = cli.Command{
+		Action: dbStats,
+		Name:   "stats",
+		Usage:  "Print leveldb statistics",
+	}
+	dbCompactCmd = cli.Command{
+		Action: dbCompact,
+		Name:   "compact",
+		Usage:  "Compact leveldb database. WARNING: May take a very long time",
+		Description: `This command performs a database compaction. 
+WARNING: This operation may take a very long time to finish, and may cause database
+corruption if it is aborted during execution'!`,
+	}
+	dbGetCmd = cli.Command{
+		Action:      dbGet,
+		Name:        "get",
+		Usage:       "Show the value of a database key",
+		ArgsUsage:   "<hex-encoded key>",
+		Description: "This command looks up the specified database key from the database.",
+	}
+	dbDeleteCmd = cli.Command{
+		Action:    dbDelete,
+		Name:      "delete",
+		Usage:     "Delete a database key (WARNING: may corrupt your database)",
+		ArgsUsage: "<hex-encoded key>",
+		Description: `This command deletes the specified database key from the database. 
+WARNING: This is a low-level operation which may cause database corruption!`,
+	}
+	dbPutCmd = cli.Command{
+		Action:    dbPut,
+		Name:      "put",
+		Usage:     "Set the value of a database key (WARNING: may corrupt your database)",
+		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
+		Description: `This command sets a given database key to the given value. 
+WARNING: This is a low-level operation which may cause database corruption!`,
+	}
+)
+
+func removeDB(ctx *cli.Context) error {
+	stack, config := makeConfigNode(ctx)
+
+	// Remove the full node state database
+	path := stack.ResolvePath("chaindata")
+	if common.FileExist(path) {
+		confirmAndRemoveDB(path, "full node state database")
+	} else {
+		log.Info("Full node state database missing", "path", path)
+	}
+	// Remove the full node ancient database
+	path = config.Eth.DatabaseFreezer
+	switch {
+	case path == "":
+		path = filepath.Join(stack.ResolvePath("chaindata"), "ancient")
+	case !filepath.IsAbs(path):
+		path = config.Node.ResolvePath(path)
+	}
+	if common.FileExist(path) {
+		confirmAndRemoveDB(path, "full node ancient database")
+	} else {
+		log.Info("Full node ancient database missing", "path", path)
+	}
+	// Remove the light node database
+	path = stack.ResolvePath("lightchaindata")
+	if common.FileExist(path) {
+		confirmAndRemoveDB(path, "light node database")
+	} else {
+		log.Info("Light node database missing", "path", path)
+	}
+	return nil
+}
+
+// confirmAndRemoveDB prompts the user for a last confirmation and removes the
+// folder if accepted.
+func confirmAndRemoveDB(database string, kind string) {
+	confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database))
+	switch {
+	case err != nil:
+		utils.Fatalf("%v", err)
+	case !confirm:
+		log.Info("Database deletion skipped", "path", database)
+	default:
+		start := time.Now()
+		filepath.Walk(database, func(path string, info os.FileInfo, err error) error {
+			// If we're at the top level folder, recurse into
+			if path == database {
+				return nil
+			}
+			// Delete all the files, but not subfolders
+			if !info.IsDir() {
+				os.Remove(path)
+				return nil
+			}
+			return filepath.SkipDir
+		})
+		log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start)))
+	}
+}
+
+func inspect(ctx *cli.Context) error {
+	var (
+		prefix []byte
+		start  []byte
+	)
+	if ctx.NArg() > 2 {
+		return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage)
+	}
+	if ctx.NArg() >= 1 {
+		if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
+			return fmt.Errorf("failed to hex-decode 'prefix': %v", err)
+		} else {
+			prefix = d
+		}
+	}
+	if ctx.NArg() >= 2 {
+		if d, err := hexutil.Decode(ctx.Args().Get(1)); err != nil {
+			return fmt.Errorf("failed to hex-decode 'start': %v", err)
+		} else {
+			start = d
+		}
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	_, chainDb := utils.MakeChain(ctx, stack, true)
+	defer chainDb.Close()
+
+	return rawdb.InspectDatabase(chainDb, prefix, start)
+}
+
+func showLeveldbStats(db ethdb.Stater) {
+	if stats, err := db.Stat("leveldb.stats"); err != nil {
+		log.Warn("Failed to read database stats", "error", err)
+	} else {
+		fmt.Println(stats)
+	}
+	if ioStats, err := db.Stat("leveldb.iostats"); err != nil {
+		log.Warn("Failed to read database iostats", "error", err)
+	} else {
+		fmt.Println(ioStats)
+	}
+}
+
+func dbStats(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	path := stack.ResolvePath("chaindata")
+	db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
+		options.ReadOnly = true
+	})
+	if err != nil {
+		return err
+	}
+	showLeveldbStats(db)
+	err = db.Close()
+	if err != nil {
+		log.Info("Close err", "error", err)
+	}
+	return nil
+}
+
+func dbCompact(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	path := stack.ResolvePath("chaindata")
+	cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100
+	db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
+		options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles()
+		options.BlockCacheCapacity = cache / 2 * opt.MiB
+		options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
+	})
+	if err != nil {
+		return err
+	}
+	showLeveldbStats(db)
+	log.Info("Triggering compaction")
+	err = db.Compact(nil, nil)
+	if err != nil {
+		log.Info("Compact err", "error", err)
+	}
+	showLeveldbStats(db)
+	log.Info("Closing db")
+	err = db.Close()
+	if err != nil {
+		log.Info("Close err", "error", err)
+	}
+	log.Info("Exiting")
+	return err
+}
+
+// dbGet shows the value of a given database key
+func dbGet(ctx *cli.Context) error {
+	if ctx.NArg() != 1 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	path := stack.ResolvePath("chaindata")
+	db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
+		options.ReadOnly = true
+	})
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	key, err := hexutil.Decode(ctx.Args().Get(0))
+	if err != nil {
+		log.Info("Could not decode the key", "error", err)
+		return err
+	}
+	data, err := db.Get(key)
+	if err != nil {
+		log.Info("Get operation failed", "error", err)
+		return err
+	}
+	fmt.Printf("key %#x:\n\t%#x\n", key, data)
+	return nil
+}
+
+// dbDelete deletes a key from the database
+func dbDelete(ctx *cli.Context) error {
+	if ctx.NArg() != 1 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack)
+	defer db.Close()
+	key, err := hexutil.Decode(ctx.Args().Get(0))
+	if err != nil {
+		log.Info("Could not decode the key", "error", err)
+		return err
+	}
+	if err = db.Delete(key); err != nil {
+		log.Info("Delete operation returned an error", "error", err)
+		return err
+	}
+	return nil
+}
+
+// dbPut overwrite a value in the database
+func dbPut(ctx *cli.Context) error {
+	if ctx.NArg() != 2 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack)
+	defer db.Close()
+	var (
+		key   []byte
+		value []byte
+		data  []byte
+		err   error
+	)
+	key, err = hexutil.Decode(ctx.Args().Get(0))
+	if err != nil {
+		log.Info("Could not decode the key", "error", err)
+		return err
+	}
+	value, err = hexutil.Decode(ctx.Args().Get(1))
+	if err != nil {
+		log.Info("Could not decode the value", "error", err)
+		return err
+	}
+	data, err = db.Get(key)
+	if err == nil {
+		fmt.Printf("Previous value:\n%#x\n", data)
+	}
+	return db.Put(key, value)
+}

cmd/geth/genesis_test.go

@@ -81,10 +81,10 @@ func TestCustomGenesis(t *testing.T) {
 		if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil {
 			t.Fatalf("test %d: failed to write genesis file: %v", i, err)
 		}
-		runGeth(t, "--nousb", "--datadir", datadir, "init", json).WaitExit()
+		runGeth(t, "--datadir", datadir, "init", json).WaitExit()
 
 		// Query the custom genesis block
-		geth := runGeth(t, "--nousb",
+		geth := runGeth(t, "--networkid", "1337", "--syncmode=full",
 			"--datadir", datadir, "--maxpeers", "0", "--port", "0",
 			"--nodiscover", "--nat", "none", "--ipcdisable",
 			"--exec", tt.query, "console")

cmd/geth/les_test.go

@@ -2,7 +2,12 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"path/filepath"
+	"runtime"
+	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -95,24 +100,57 @@ func (g *gethrpc) waitSynced() {
 	}
 }
 
+// ipcEndpoint resolves an IPC endpoint based on a configured value, taking into
+// account the set data folders as well as the designated platform we're currently
+// running on.
+func ipcEndpoint(ipcPath, datadir string) string {
+	// On windows we can only use plain top-level pipes
+	if runtime.GOOS == "windows" {
+		if strings.HasPrefix(ipcPath, `\\.\pipe\`) {
+			return ipcPath
+		}
+		return `\\.\pipe\` + ipcPath
+	}
+	// Resolve names into the data directory full paths otherwise
+	if filepath.Base(ipcPath) == ipcPath {
+		if datadir == "" {
+			return filepath.Join(os.TempDir(), ipcPath)
+		}
+		return filepath.Join(datadir, ipcPath)
+	}
+	return ipcPath
+}
+
+// nextIPC ensures that each ipc pipe gets a unique name.
+// On linux, it works well to use ipc pipes all over the filesystem (in datadirs),
+// but windows require pipes to sit in "\\.\pipe\". Therefore, to run several
+// nodes simultaneously, we need to distinguish between them, which we do by
+// the pipe filename instead of folder.
+var nextIPC = uint32(0)
+
 func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc {
-	g := &gethrpc{name: name}
-	args = append([]string{"--networkid=42", "--port=0", "--nousb"}, args...)
+	ipcName := fmt.Sprintf("geth-%d.ipc", atomic.AddUint32(&nextIPC, 1))
+	args = append([]string{"--networkid=42", "--port=0", "--ipcpath", ipcName}, args...)
 	t.Logf("Starting %v with rpc: %v", name, args)
-	g.geth = runGeth(t, args...)
+
+	g := &gethrpc{
+		name: name,
+		geth: runGeth(t, args...),
+	}
 	// wait before we can attach to it. TODO: probe for it properly
 	time.Sleep(1 * time.Second)
 	var err error
-	ipcpath := filepath.Join(g.geth.Datadir, "geth.ipc")
-	g.rpc, err = rpc.Dial(ipcpath)
-	if err != nil {
-		t.Fatalf("%v rpc connect: %v", name, err)
+	ipcpath := ipcEndpoint(ipcName, g.geth.Datadir)
+	if g.rpc, err = rpc.Dial(ipcpath); err != nil {
+		t.Fatalf("%v rpc connect to %v: %v", name, ipcpath, err)
 	}
 	return g
 }
 
 func initGeth(t *testing.T) string {
-	g := runGeth(t, "--nousb", "--networkid=42", "init", "./testdata/clique.json")
+	args := []string{"--networkid=42", "init", "./testdata/clique.json"}
+	t.Logf("Initializing geth: %v ", args)
+	g := runGeth(t, args...)
 	datadir := g.Datadir
 	g.WaitExit()
 	return datadir
@@ -120,15 +158,16 @@ func initGeth(t *testing.T) string {
 
 func startLightServer(t *testing.T) *gethrpc {
 	datadir := initGeth(t)
-	runGeth(t, "--nousb", "--datadir", datadir, "--password", "./testdata/password.txt", "account", "import", "./testdata/key.prv").WaitExit()
+	t.Logf("Importing keys to geth")
+	runGeth(t, "--datadir", datadir, "--password", "./testdata/password.txt", "account", "import", "./testdata/key.prv", "--lightkdf").WaitExit()
 	account := "0x02f0d131f1f97aef08aec6e3291b957d9efe7105"
-	server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--mine", "--light.serve=100", "--light.maxpeers=1", "--nodiscover", "--nat=extip:127.0.0.1")
+	server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--mine", "--light.serve=100", "--light.maxpeers=1", "--nodiscover", "--nat=extip:127.0.0.1", "--verbosity=4")
 	return server
 }
 
 func startClient(t *testing.T, name string) *gethrpc {
 	datadir := initGeth(t)
-	return startGethWithIpc(t, name, "--datadir", datadir, "--nodiscover", "--syncmode=light", "--nat=extip:127.0.0.1")
+	return startGethWithIpc(t, name, "--datadir", datadir, "--nodiscover", "--syncmode=light", "--nat=extip:127.0.0.1", "--verbosity=4")
 }
 
 func TestPriorityClient(t *testing.T) {
@@ -151,7 +190,7 @@ func TestPriorityClient(t *testing.T) {
 	prioCli := startClient(t, "prioCli")
 	defer prioCli.killAndWait()
 	// 3_000_000_000 once we move to Go 1.13
-	tokens := 3000000000
+	tokens := uint64(3000000000)
 	lightServer.callRPC(nil, "les_addBalance", prioCli.getNodeInfo().ID, tokens)
 	prioCli.addPeer(lightServer)
 

cmd/geth/main.go

@@ -42,7 +42,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	gopsutil "github.com/shirou/gopsutil/mem"
-	cli "gopkg.in/urfave/cli.v1"
+	"gopkg.in/urfave/cli.v1"
 )
 
 const (
@@ -61,14 +61,15 @@ var (
 		utils.UnlockedAccountFlag,
 		utils.PasswordFileFlag,
 		utils.BootnodesFlag,
-		utils.LegacyBootnodesV4Flag,
-		utils.LegacyBootnodesV5Flag,
 		utils.DataDirFlag,
 		utils.AncientFlag,
+		utils.MinFreeDiskSpaceFlag,
 		utils.KeyStoreDirFlag,
 		utils.ExternalSignerFlag,
 		utils.NoUSBFlag,
+		utils.USBFlag,
 		utils.SmartCardDaemonPathFlag,
+		utils.OverrideBerlinFlag,
 		utils.EthashCacheDirFlag,
 		utils.EthashCachesInMemoryFlag,
 		utils.EthashCachesOnDiskFlag,
@@ -94,17 +95,17 @@ var (
 		utils.SnapshotFlag,
 		utils.TxLookupLimitFlag,
 		utils.LightServeFlag,
-		utils.LegacyLightServFlag,
 		utils.LightIngressFlag,
 		utils.LightEgressFlag,
 		utils.LightMaxPeersFlag,
-		utils.LegacyLightPeersFlag,
 		utils.LightNoPruneFlag,
 		utils.LightKDFFlag,
 		utils.UltraLightServersFlag,
 		utils.UltraLightFractionFlag,
 		utils.UltraLightOnlyAnnounceFlag,
+		utils.LightNoSyncServeFlag,
 		utils.WhitelistFlag,
+		utils.BloomFilterSizeFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
 		utils.CacheTrieFlag,
@@ -113,22 +114,18 @@ var (
 		utils.CacheGCFlag,
 		utils.CacheSnapshotFlag,
 		utils.CacheNoPrefetchFlag,
+		utils.CachePreimagesFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,
 		utils.MiningEnabledFlag,
 		utils.MinerThreadsFlag,
-		utils.LegacyMinerThreadsFlag,
 		utils.MinerNotifyFlag,
 		utils.MinerGasTargetFlag,
-		utils.LegacyMinerGasTargetFlag,
 		utils.MinerGasLimitFlag,
 		utils.MinerGasPriceFlag,
-		utils.LegacyMinerGasPriceFlag,
 		utils.MinerEtherbaseFlag,
-		utils.LegacyMinerEtherbaseFlag,
 		utils.MinerExtraDataFlag,
-		utils.LegacyMinerExtraDataFlag,
 		utils.MinerRecommitIntervalFlag,
 		utils.MinerNoVerfiyFlag,
 		utils.NATFlag,
@@ -138,22 +135,20 @@ var (
 		utils.NodeKeyFileFlag,
 		utils.NodeKeyHexFlag,
 		utils.DNSDiscoveryFlag,
+		utils.MainnetFlag,
 		utils.DeveloperFlag,
 		utils.DeveloperPeriodFlag,
-		utils.LegacyTestnetFlag,
 		utils.RopstenFlag,
 		utils.RinkebyFlag,
 		utils.GoerliFlag,
-		utils.YoloV1Flag,
+		utils.YoloV3Flag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,
 		utils.FakePoWFlag,
 		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
-		utils.LegacyGpoBlocksFlag,
 		utils.GpoPercentileFlag,
-		utils.LegacyGpoPercentileFlag,
 		utils.GpoMaxGasPriceFlag,
 		utils.EWASMInterpreterFlag,
 		utils.EVMInterpreterFlag,
@@ -171,32 +166,24 @@ var (
 		utils.LegacyRPCPortFlag,
 		utils.LegacyRPCCORSDomainFlag,
 		utils.LegacyRPCVirtualHostsFlag,
+		utils.LegacyRPCApiFlag,
 		utils.GraphQLEnabledFlag,
 		utils.GraphQLCORSDomainFlag,
 		utils.GraphQLVirtualHostsFlag,
 		utils.HTTPApiFlag,
-		utils.LegacyRPCApiFlag,
+		utils.HTTPPathPrefixFlag,
 		utils.WSEnabledFlag,
 		utils.WSListenAddrFlag,
-		utils.LegacyWSListenAddrFlag,
 		utils.WSPortFlag,
-		utils.LegacyWSPortFlag,
 		utils.WSApiFlag,
-		utils.LegacyWSApiFlag,
 		utils.WSAllowedOriginsFlag,
-		utils.LegacyWSAllowedOriginsFlag,
+		utils.WSPathPrefixFlag,
 		utils.IPCDisabledFlag,
 		utils.IPCPathFlag,
 		utils.InsecureUnlockAllowedFlag,
 		utils.RPCGlobalGasCapFlag,
 		utils.RPCGlobalTxFeeCapFlag,
-	}
-
-	whisperFlags = []cli.Flag{
-		utils.WhisperEnabledFlag,
-		utils.WhisperMaxMessageSizeFlag,
-		utils.WhisperMinPOWFlag,
-		utils.WhisperRestrictConnectionBetweenLightClientsFlag,
+		utils.AllowUnprotectedTxs,
 	}
 
 	metricsFlags = []cli.Flag{
@@ -217,7 +204,7 @@ func init() {
 	// Initialize the CLI app and start Geth
 	app.Action = geth
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2020 The go-ethereum Authors"
+	app.Copyright = "Copyright 2013-2021 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
@@ -229,7 +216,6 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
-		inspectCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,
@@ -241,13 +227,16 @@ func init() {
 		makecacheCommand,
 		makedagCommand,
 		versionCommand,
+		versionCheckCommand,
 		licenseCommand,
 		// See config.go
 		dumpConfigCommand,
-		// See retesteth.go
-		retestethCommand,
+		// see dbcmd.go
+		dbCommand,
 		// See cmd/utils/flags_legacy.go
 		utils.ShowDeprecated,
+		// See snapshot.go
+		snapshotCommand,
 	}
 	sort.Sort(cli.CommandsByName(app.Commands))
 
@@ -255,8 +244,6 @@ func init() {
 	app.Flags = append(app.Flags, rpcFlags...)
 	app.Flags = append(app.Flags, consoleFlags...)
 	app.Flags = append(app.Flags, debug.Flags...)
-	app.Flags = append(app.Flags, debug.DeprecatedFlags...)
-	app.Flags = append(app.Flags, whisperFlags...)
 	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
@@ -281,11 +268,6 @@ func main() {
 func prepare(ctx *cli.Context) {
 	// If we're running a known preset, log it for convenience.
 	switch {
-	case ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name):
-		log.Info("Starting Geth on Ropsten testnet...")
-		log.Warn("The --testnet flag is ambiguous! Please specify one of --goerli, --rinkeby, or --ropsten.")
-		log.Warn("The generic --testnet flag is deprecated and will be removed in the future!")
-
 	case ctx.GlobalIsSet(utils.RopstenFlag.Name):
 		log.Info("Starting Geth on Ropsten testnet...")
 
@@ -295,6 +277,9 @@ func prepare(ctx *cli.Context) {
 	case ctx.GlobalIsSet(utils.GoerliFlag.Name):
 		log.Info("Starting Geth on Görli testnet...")
 
+	case ctx.GlobalIsSet(utils.YoloV3Flag.Name):
+		log.Info("Starting Geth on YOLOv3 testnet...")
+
 	case ctx.GlobalIsSet(utils.DeveloperFlag.Name):
 		log.Info("Starting Geth in ephemeral dev mode...")
 
@@ -304,7 +289,7 @@ func prepare(ctx *cli.Context) {
 	// If we're a full node on mainnet without --cache specified, bump default cache allowance
 	if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
 		// Make sure we're not on any supported preconfigured testnet either
-		if !ctx.GlobalIsSet(utils.LegacyTestnetFlag.Name) && !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
+		if !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
 			// Nope, we're really on mainnet. Bump that cache up!
 			log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
 			ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))
@@ -366,7 +351,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend) {
 	debug.Memsize.Add("node", stack)
 
 	// Start up the node itself
-	utils.StartNode(stack)
+	utils.StartNode(ctx, stack)
 
 	// Unlock any account specifically requested
 	unlockAccounts(ctx, stack)
@@ -449,19 +434,11 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend) {
 		if !ok {
 			utils.Fatalf("Ethereum service not running: %v", err)
 		}
-
 		// Set the gas price to the limits from the CLI and start mining
 		gasprice := utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
-		if ctx.GlobalIsSet(utils.LegacyMinerGasPriceFlag.Name) && !ctx.GlobalIsSet(utils.MinerGasPriceFlag.Name) {
-			gasprice = utils.GlobalBig(ctx, utils.LegacyMinerGasPriceFlag.Name)
-		}
 		ethBackend.TxPool().SetGasPrice(gasprice)
 		// start mining
 		threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name)
-		if ctx.GlobalIsSet(utils.LegacyMinerThreadsFlag.Name) && !ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) {
-			threads = ctx.GlobalInt(utils.LegacyMinerThreadsFlag.Name)
-			log.Warn("The flag --minerthreads is deprecated and will be removed in the future, please use --miner.threads")
-		}
 		if err := ethBackend.StartMining(threads); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}

cmd/geth/misccmd.go

@@ -25,12 +25,23 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
-	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/params"
 	"gopkg.in/urfave/cli.v1"
 )
 
 var (
+	VersionCheckUrlFlag = cli.StringFlag{
+		Name:  "check.url",
+		Usage: "URL to use when checking vulnerabilities",
+		Value: "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json",
+	}
+	VersionCheckVersionFlag = cli.StringFlag{
+		Name:  "check.version",
+		Usage: "Version to check",
+		Value: fmt.Sprintf("Geth/v%v/%v-%v/%v",
+			params.VersionWithCommit(gitCommit, gitDate),
+			runtime.GOOS, runtime.GOARCH, runtime.Version()),
+	}
 	makecacheCommand = cli.Command{
 		Action:    utils.MigrateFlags(makecache),
 		Name:      "makecache",
@@ -65,6 +76,21 @@ Regular users do not need to execute it.
 		Category:  "MISCELLANEOUS COMMANDS",
 		Description: `
 The output of this command is supposed to be machine-readable.
+`,
+	}
+	versionCheckCommand = cli.Command{
+		Action: utils.MigrateFlags(versionCheck),
+		Flags: []cli.Flag{
+			VersionCheckUrlFlag,
+			VersionCheckVersionFlag,
+		},
+		Name:      "version-check",
+		Usage:     "Checks (online) whether the current version suffers from any known security vulnerabilities",
+		ArgsUsage: "<versionstring (optional)>",
+		Category:  "MISCELLANEOUS COMMANDS",
+		Description: `
+The version-check command fetches vulnerability-information from https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json, 
+and displays information about any security vulnerabilities that affect the currently executing version.
 `,
 	}
 	licenseCommand = cli.Command{
@@ -116,7 +142,6 @@ func version(ctx *cli.Context) error {
 		fmt.Println("Git Commit Date:", gitDate)
 	}
 	fmt.Println("Architecture:", runtime.GOARCH)
-	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
 	fmt.Println("Go Version:", runtime.Version())
 	fmt.Println("Operating System:", runtime.GOOS)
 	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))

cmd/geth/retesteth.go

@@ -1,916 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"math/big"
-	"os"
-	"os/signal"
-	"time"
-
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/ethereum/go-ethereum/consensus"
-	"github.com/ethereum/go-ethereum/consensus/ethash"
-	"github.com/ethereum/go-ethereum/consensus/misc"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/trie"
-
-	cli "gopkg.in/urfave/cli.v1"
-)
-
-var (
-	rpcPortFlag = cli.IntFlag{
-		Name:  "rpcport",
-		Usage: "HTTP-RPC server listening port",
-		Value: node.DefaultHTTPPort,
-	}
-	retestethCommand = cli.Command{
-		Action:      utils.MigrateFlags(retesteth),
-		Name:        "retesteth",
-		Usage:       "Launches geth in retesteth mode",
-		ArgsUsage:   "",
-		Flags:       []cli.Flag{rpcPortFlag},
-		Category:    "MISCELLANEOUS COMMANDS",
-		Description: `Launches geth in retesteth mode (no database, no network, only retesteth RPC interface)`,
-	}
-)
-
-type RetestethTestAPI interface {
-	SetChainParams(ctx context.Context, chainParams ChainParams) (bool, error)
-	MineBlocks(ctx context.Context, number uint64) (bool, error)
-	ModifyTimestamp(ctx context.Context, interval uint64) (bool, error)
-	ImportRawBlock(ctx context.Context, rawBlock hexutil.Bytes) (common.Hash, error)
-	RewindToBlock(ctx context.Context, number uint64) (bool, error)
-	GetLogHash(ctx context.Context, txHash common.Hash) (common.Hash, error)
-}
-
-type RetestethEthAPI interface {
-	SendRawTransaction(ctx context.Context, rawTx hexutil.Bytes) (common.Hash, error)
-	BlockNumber(ctx context.Context) (uint64, error)
-	GetBlockByNumber(ctx context.Context, blockNr math.HexOrDecimal64, fullTx bool) (map[string]interface{}, error)
-	GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error)
-	GetBalance(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (*math.HexOrDecimal256, error)
-	GetCode(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (hexutil.Bytes, error)
-	GetTransactionCount(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (uint64, error)
-}
-
-type RetestethDebugAPI interface {
-	AccountRange(ctx context.Context,
-		blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
-		addressHash *math.HexOrDecimal256, maxResults uint64,
-	) (AccountRangeResult, error)
-	StorageRangeAt(ctx context.Context,
-		blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
-		address common.Address,
-		begin *math.HexOrDecimal256, maxResults uint64,
-	) (StorageRangeResult, error)
-}
-
-type RetestWeb3API interface {
-	ClientVersion(ctx context.Context) (string, error)
-}
-
-type RetestethAPI struct {
-	ethDb         ethdb.Database
-	db            state.Database
-	chainConfig   *params.ChainConfig
-	author        common.Address
-	extraData     []byte
-	genesisHash   common.Hash
-	engine        *NoRewardEngine
-	blockchain    *core.BlockChain
-	txMap         map[common.Address]map[uint64]*types.Transaction // Sender -> Nonce -> Transaction
-	txSenders     map[common.Address]struct{}                      // Set of transaction senders
-	blockInterval uint64
-}
-
-type ChainParams struct {
-	SealEngine string                            `json:"sealEngine"`
-	Params     CParamsParams                     `json:"params"`
-	Genesis    CParamsGenesis                    `json:"genesis"`
-	Accounts   map[common.Address]CParamsAccount `json:"accounts"`
-}
-
-type CParamsParams struct {
-	AccountStartNonce          math.HexOrDecimal64   `json:"accountStartNonce"`
-	HomesteadForkBlock         *math.HexOrDecimal64  `json:"homesteadForkBlock"`
-	EIP150ForkBlock            *math.HexOrDecimal64  `json:"EIP150ForkBlock"`
-	EIP158ForkBlock            *math.HexOrDecimal64  `json:"EIP158ForkBlock"`
-	DaoHardforkBlock           *math.HexOrDecimal64  `json:"daoHardforkBlock"`
-	ByzantiumForkBlock         *math.HexOrDecimal64  `json:"byzantiumForkBlock"`
-	ConstantinopleForkBlock    *math.HexOrDecimal64  `json:"constantinopleForkBlock"`
-	ConstantinopleFixForkBlock *math.HexOrDecimal64  `json:"constantinopleFixForkBlock"`
-	IstanbulBlock              *math.HexOrDecimal64  `json:"istanbulForkBlock"`
-	ChainID                    *math.HexOrDecimal256 `json:"chainID"`
-	MaximumExtraDataSize       math.HexOrDecimal64   `json:"maximumExtraDataSize"`
-	TieBreakingGas             bool                  `json:"tieBreakingGas"`
-	MinGasLimit                math.HexOrDecimal64   `json:"minGasLimit"`
-	MaxGasLimit                math.HexOrDecimal64   `json:"maxGasLimit"`
-	GasLimitBoundDivisor       math.HexOrDecimal64   `json:"gasLimitBoundDivisor"`
-	MinimumDifficulty          math.HexOrDecimal256  `json:"minimumDifficulty"`
-	DifficultyBoundDivisor     math.HexOrDecimal256  `json:"difficultyBoundDivisor"`
-	DurationLimit              math.HexOrDecimal256  `json:"durationLimit"`
-	BlockReward                math.HexOrDecimal256  `json:"blockReward"`
-	NetworkID                  math.HexOrDecimal256  `json:"networkID"`
-}
-
-type CParamsGenesis struct {
-	Nonce      math.HexOrDecimal64   `json:"nonce"`
-	Difficulty *math.HexOrDecimal256 `json:"difficulty"`
-	MixHash    *math.HexOrDecimal256 `json:"mixHash"`
-	Author     common.Address        `json:"author"`
-	Timestamp  math.HexOrDecimal64   `json:"timestamp"`
-	ParentHash common.Hash           `json:"parentHash"`
-	ExtraData  hexutil.Bytes         `json:"extraData"`
-	GasLimit   math.HexOrDecimal64   `json:"gasLimit"`
-}
-
-type CParamsAccount struct {
-	Balance     *math.HexOrDecimal256 `json:"balance"`
-	Precompiled *CPAccountPrecompiled `json:"precompiled"`
-	Code        hexutil.Bytes         `json:"code"`
-	Storage     map[string]string     `json:"storage"`
-	Nonce       *math.HexOrDecimal64  `json:"nonce"`
-}
-
-type CPAccountPrecompiled struct {
-	Name          string                `json:"name"`
-	StartingBlock math.HexOrDecimal64   `json:"startingBlock"`
-	Linear        *CPAPrecompiledLinear `json:"linear"`
-}
-
-type CPAPrecompiledLinear struct {
-	Base uint64 `json:"base"`
-	Word uint64 `json:"word"`
-}
-
-type AccountRangeResult struct {
-	AddressMap map[common.Hash]common.Address `json:"addressMap"`
-	NextKey    common.Hash                    `json:"nextKey"`
-}
-
-type StorageRangeResult struct {
-	Complete bool                   `json:"complete"`
-	Storage  map[common.Hash]SRItem `json:"storage"`
-}
-
-type SRItem struct {
-	Key   string `json:"key"`
-	Value string `json:"value"`
-}
-
-type NoRewardEngine struct {
-	inner     consensus.Engine
-	rewardsOn bool
-}
-
-func (e *NoRewardEngine) Author(header *types.Header) (common.Address, error) {
-	return e.inner.Author(header)
-}
-
-func (e *NoRewardEngine) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error {
-	return e.inner.VerifyHeader(chain, header, seal)
-}
-
-func (e *NoRewardEngine) VerifyHeaders(chain consensus.ChainHeaderReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) {
-	return e.inner.VerifyHeaders(chain, headers, seals)
-}
-
-func (e *NoRewardEngine) VerifyUncles(chain consensus.ChainReader, block *types.Block) error {
-	return e.inner.VerifyUncles(chain, block)
-}
-
-func (e *NoRewardEngine) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error {
-	return e.inner.VerifySeal(chain, header)
-}
-
-func (e *NoRewardEngine) Prepare(chain consensus.ChainHeaderReader, header *types.Header) error {
-	return e.inner.Prepare(chain, header)
-}
-
-func (e *NoRewardEngine) accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) {
-	// Simply touch miner and uncle coinbase accounts
-	reward := big.NewInt(0)
-	for _, uncle := range uncles {
-		state.AddBalance(uncle.Coinbase, reward)
-	}
-	state.AddBalance(header.Coinbase, reward)
-}
-
-func (e *NoRewardEngine) Finalize(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB, txs []*types.Transaction,
-	uncles []*types.Header) {
-	if e.rewardsOn {
-		e.inner.Finalize(chain, header, statedb, txs, uncles)
-	} else {
-		e.accumulateRewards(chain.Config(), statedb, header, uncles)
-		header.Root = statedb.IntermediateRoot(chain.Config().IsEIP158(header.Number))
-	}
-}
-
-func (e *NoRewardEngine) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, statedb *state.StateDB, txs []*types.Transaction,
-	uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) {
-	if e.rewardsOn {
-		return e.inner.FinalizeAndAssemble(chain, header, statedb, txs, uncles, receipts)
-	} else {
-		e.accumulateRewards(chain.Config(), statedb, header, uncles)
-		header.Root = statedb.IntermediateRoot(chain.Config().IsEIP158(header.Number))
-
-		// Header seems complete, assemble into a block and return
-		return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie)), nil
-	}
-}
-
-func (e *NoRewardEngine) Seal(chain consensus.ChainHeaderReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error {
-	return e.inner.Seal(chain, block, results, stop)
-}
-
-func (e *NoRewardEngine) SealHash(header *types.Header) common.Hash {
-	return e.inner.SealHash(header)
-}
-
-func (e *NoRewardEngine) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int {
-	return e.inner.CalcDifficulty(chain, time, parent)
-}
-
-func (e *NoRewardEngine) APIs(chain consensus.ChainHeaderReader) []rpc.API {
-	return e.inner.APIs(chain)
-}
-
-func (e *NoRewardEngine) Close() error {
-	return e.inner.Close()
-}
-
-func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainParams) (bool, error) {
-	// Clean up
-	if api.blockchain != nil {
-		api.blockchain.Stop()
-	}
-	if api.engine != nil {
-		api.engine.Close()
-	}
-	if api.ethDb != nil {
-		api.ethDb.Close()
-	}
-	ethDb := rawdb.NewMemoryDatabase()
-	accounts := make(core.GenesisAlloc)
-	for address, account := range chainParams.Accounts {
-		balance := big.NewInt(0)
-		if account.Balance != nil {
-			balance.Set((*big.Int)(account.Balance))
-		}
-		var nonce uint64
-		if account.Nonce != nil {
-			nonce = uint64(*account.Nonce)
-		}
-		if account.Precompiled == nil || account.Balance != nil {
-			storage := make(map[common.Hash]common.Hash)
-			for k, v := range account.Storage {
-				storage[common.HexToHash(k)] = common.HexToHash(v)
-			}
-			accounts[address] = core.GenesisAccount{
-				Balance: balance,
-				Code:    account.Code,
-				Nonce:   nonce,
-				Storage: storage,
-			}
-		}
-	}
-	chainId := big.NewInt(1)
-	if chainParams.Params.ChainID != nil {
-		chainId.Set((*big.Int)(chainParams.Params.ChainID))
-	}
-	var (
-		homesteadBlock      *big.Int
-		daoForkBlock        *big.Int
-		eip150Block         *big.Int
-		eip155Block         *big.Int
-		eip158Block         *big.Int
-		byzantiumBlock      *big.Int
-		constantinopleBlock *big.Int
-		petersburgBlock     *big.Int
-		istanbulBlock       *big.Int
-	)
-	if chainParams.Params.HomesteadForkBlock != nil {
-		homesteadBlock = big.NewInt(int64(*chainParams.Params.HomesteadForkBlock))
-	}
-	if chainParams.Params.DaoHardforkBlock != nil {
-		daoForkBlock = big.NewInt(int64(*chainParams.Params.DaoHardforkBlock))
-	}
-	if chainParams.Params.EIP150ForkBlock != nil {
-		eip150Block = big.NewInt(int64(*chainParams.Params.EIP150ForkBlock))
-	}
-	if chainParams.Params.EIP158ForkBlock != nil {
-		eip158Block = big.NewInt(int64(*chainParams.Params.EIP158ForkBlock))
-		eip155Block = eip158Block
-	}
-	if chainParams.Params.ByzantiumForkBlock != nil {
-		byzantiumBlock = big.NewInt(int64(*chainParams.Params.ByzantiumForkBlock))
-	}
-	if chainParams.Params.ConstantinopleForkBlock != nil {
-		constantinopleBlock = big.NewInt(int64(*chainParams.Params.ConstantinopleForkBlock))
-	}
-	if chainParams.Params.ConstantinopleFixForkBlock != nil {
-		petersburgBlock = big.NewInt(int64(*chainParams.Params.ConstantinopleFixForkBlock))
-	}
-	if constantinopleBlock != nil && petersburgBlock == nil {
-		petersburgBlock = big.NewInt(100000000000)
-	}
-	if chainParams.Params.IstanbulBlock != nil {
-		istanbulBlock = big.NewInt(int64(*chainParams.Params.IstanbulBlock))
-	}
-
-	genesis := &core.Genesis{
-		Config: &params.ChainConfig{
-			ChainID:             chainId,
-			HomesteadBlock:      homesteadBlock,
-			DAOForkBlock:        daoForkBlock,
-			DAOForkSupport:      true,
-			EIP150Block:         eip150Block,
-			EIP155Block:         eip155Block,
-			EIP158Block:         eip158Block,
-			ByzantiumBlock:      byzantiumBlock,
-			ConstantinopleBlock: constantinopleBlock,
-			PetersburgBlock:     petersburgBlock,
-			IstanbulBlock:       istanbulBlock,
-		},
-		Nonce:      uint64(chainParams.Genesis.Nonce),
-		Timestamp:  uint64(chainParams.Genesis.Timestamp),
-		ExtraData:  chainParams.Genesis.ExtraData,
-		GasLimit:   uint64(chainParams.Genesis.GasLimit),
-		Difficulty: big.NewInt(0).Set((*big.Int)(chainParams.Genesis.Difficulty)),
-		Mixhash:    common.BigToHash((*big.Int)(chainParams.Genesis.MixHash)),
-		Coinbase:   chainParams.Genesis.Author,
-		ParentHash: chainParams.Genesis.ParentHash,
-		Alloc:      accounts,
-	}
-	chainConfig, genesisHash, err := core.SetupGenesisBlock(ethDb, genesis)
-	if err != nil {
-		return false, err
-	}
-	fmt.Printf("Chain config: %v\n", chainConfig)
-
-	var inner consensus.Engine
-	switch chainParams.SealEngine {
-	case "NoProof", "NoReward":
-		inner = ethash.NewFaker()
-	case "Ethash":
-		inner = ethash.New(ethash.Config{
-			CacheDir:         "ethash",
-			CachesInMem:      2,
-			CachesOnDisk:     3,
-			CachesLockMmap:   false,
-			DatasetsInMem:    1,
-			DatasetsOnDisk:   2,
-			DatasetsLockMmap: false,
-		}, nil, false)
-	default:
-		return false, fmt.Errorf("unrecognised seal engine: %s", chainParams.SealEngine)
-	}
-	engine := &NoRewardEngine{inner: inner, rewardsOn: chainParams.SealEngine != "NoReward"}
-
-	blockchain, err := core.NewBlockChain(ethDb, nil, chainConfig, engine, vm.Config{}, nil, nil)
-	if err != nil {
-		return false, err
-	}
-
-	api.chainConfig = chainConfig
-	api.genesisHash = genesisHash
-	api.author = chainParams.Genesis.Author
-	api.extraData = chainParams.Genesis.ExtraData
-	api.ethDb = ethDb
-	api.engine = engine
-	api.blockchain = blockchain
-	api.db = state.NewDatabase(api.ethDb)
-	api.txMap = make(map[common.Address]map[uint64]*types.Transaction)
-	api.txSenders = make(map[common.Address]struct{})
-	api.blockInterval = 0
-	return true, nil
-}
-
-func (api *RetestethAPI) SendRawTransaction(ctx context.Context, rawTx hexutil.Bytes) (common.Hash, error) {
-	tx := new(types.Transaction)
-	if err := rlp.DecodeBytes(rawTx, tx); err != nil {
-		// Return nil is not by mistake - some tests include sending transaction where gasLimit overflows uint64
-		return common.Hash{}, nil
-	}
-	signer := types.MakeSigner(api.chainConfig, big.NewInt(int64(api.currentNumber())))
-	sender, err := types.Sender(signer, tx)
-	if err != nil {
-		return common.Hash{}, err
-	}
-	if nonceMap, ok := api.txMap[sender]; ok {
-		nonceMap[tx.Nonce()] = tx
-	} else {
-		nonceMap = make(map[uint64]*types.Transaction)
-		nonceMap[tx.Nonce()] = tx
-		api.txMap[sender] = nonceMap
-	}
-	api.txSenders[sender] = struct{}{}
-	return tx.Hash(), nil
-}
-
-func (api *RetestethAPI) MineBlocks(ctx context.Context, number uint64) (bool, error) {
-	for i := 0; i < int(number); i++ {
-		if err := api.mineBlock(); err != nil {
-			return false, err
-		}
-	}
-	fmt.Printf("Mined %d blocks\n", number)
-	return true, nil
-}
-
-func (api *RetestethAPI) currentNumber() uint64 {
-	if current := api.blockchain.CurrentBlock(); current != nil {
-		return current.NumberU64()
-	}
-	return 0
-}
-
-func (api *RetestethAPI) mineBlock() error {
-	number := api.currentNumber()
-	parentHash := rawdb.ReadCanonicalHash(api.ethDb, number)
-	parent := rawdb.ReadBlock(api.ethDb, parentHash, number)
-	var timestamp uint64
-	if api.blockInterval == 0 {
-		timestamp = uint64(time.Now().Unix())
-	} else {
-		timestamp = parent.Time() + api.blockInterval
-	}
-	gasLimit := core.CalcGasLimit(parent, 9223372036854775807, 9223372036854775807)
-	header := &types.Header{
-		ParentHash: parent.Hash(),
-		Number:     big.NewInt(int64(number + 1)),
-		GasLimit:   gasLimit,
-		Extra:      api.extraData,
-		Time:       timestamp,
-	}
-	header.Coinbase = api.author
-	if api.engine != nil {
-		api.engine.Prepare(api.blockchain, header)
-	}
-	// If we are care about TheDAO hard-fork check whether to override the extra-data or not
-	if daoBlock := api.chainConfig.DAOForkBlock; daoBlock != nil {
-		// Check whether the block is among the fork extra-override range
-		limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange)
-		if header.Number.Cmp(daoBlock) >= 0 && header.Number.Cmp(limit) < 0 {
-			// Depending whether we support or oppose the fork, override differently
-			if api.chainConfig.DAOForkSupport {
-				header.Extra = common.CopyBytes(params.DAOForkBlockExtra)
-			} else if bytes.Equal(header.Extra, params.DAOForkBlockExtra) {
-				header.Extra = []byte{} // If miner opposes, don't let it use the reserved extra-data
-			}
-		}
-	}
-	statedb, err := api.blockchain.StateAt(parent.Root())
-	if err != nil {
-		return err
-	}
-	if api.chainConfig.DAOForkSupport && api.chainConfig.DAOForkBlock != nil && api.chainConfig.DAOForkBlock.Cmp(header.Number) == 0 {
-		misc.ApplyDAOHardFork(statedb)
-	}
-	gasPool := new(core.GasPool).AddGas(header.GasLimit)
-	txCount := 0
-	var txs []*types.Transaction
-	var receipts []*types.Receipt
-	var blockFull = gasPool.Gas() < params.TxGas
-	for address := range api.txSenders {
-		if blockFull {
-			break
-		}
-		m := api.txMap[address]
-		for nonce := statedb.GetNonce(address); ; nonce++ {
-			if tx, ok := m[nonce]; ok {
-				// Try to apply transactions to the state
-				statedb.Prepare(tx.Hash(), common.Hash{}, txCount)
-				snap := statedb.Snapshot()
-
-				receipt, err := core.ApplyTransaction(
-					api.chainConfig,
-					api.blockchain,
-					&api.author,
-					gasPool,
-					statedb,
-					header, tx, &header.GasUsed, *api.blockchain.GetVMConfig(),
-				)
-				if err != nil {
-					statedb.RevertToSnapshot(snap)
-					break
-				}
-				txs = append(txs, tx)
-				receipts = append(receipts, receipt)
-				delete(m, nonce)
-				if len(m) == 0 {
-					// Last tx for the sender
-					delete(api.txMap, address)
-					delete(api.txSenders, address)
-				}
-				txCount++
-				if gasPool.Gas() < params.TxGas {
-					blockFull = true
-					break
-				}
-			} else {
-				break // Gap in the nonces
-			}
-		}
-	}
-	block, err := api.engine.FinalizeAndAssemble(api.blockchain, header, statedb, txs, []*types.Header{}, receipts)
-	if err != nil {
-		return err
-	}
-	return api.importBlock(block)
-}
-
-func (api *RetestethAPI) importBlock(block *types.Block) error {
-	if _, err := api.blockchain.InsertChain([]*types.Block{block}); err != nil {
-		return err
-	}
-	fmt.Printf("Imported block %d,  head is %d\n", block.NumberU64(), api.currentNumber())
-	return nil
-}
-
-func (api *RetestethAPI) ModifyTimestamp(ctx context.Context, interval uint64) (bool, error) {
-	api.blockInterval = interval
-	return true, nil
-}
-
-func (api *RetestethAPI) ImportRawBlock(ctx context.Context, rawBlock hexutil.Bytes) (common.Hash, error) {
-	block := new(types.Block)
-	if err := rlp.DecodeBytes(rawBlock, block); err != nil {
-		return common.Hash{}, err
-	}
-	fmt.Printf("Importing block %d with parent hash: %x, genesisHash: %x\n", block.NumberU64(), block.ParentHash(), api.genesisHash)
-	if err := api.importBlock(block); err != nil {
-		return common.Hash{}, err
-	}
-	return block.Hash(), nil
-}
-
-func (api *RetestethAPI) RewindToBlock(ctx context.Context, newHead uint64) (bool, error) {
-	if err := api.blockchain.SetHead(newHead); err != nil {
-		return false, err
-	}
-	// When we rewind, the transaction pool should be cleaned out.
-	api.txMap = make(map[common.Address]map[uint64]*types.Transaction)
-	api.txSenders = make(map[common.Address]struct{})
-	return true, nil
-}
-
-var emptyListHash common.Hash = common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347")
-
-func (api *RetestethAPI) GetLogHash(ctx context.Context, txHash common.Hash) (common.Hash, error) {
-	receipt, _, _, _ := rawdb.ReadReceipt(api.ethDb, txHash, api.chainConfig)
-	if receipt == nil {
-		return emptyListHash, nil
-	} else {
-		if logListRlp, err := rlp.EncodeToBytes(receipt.Logs); err != nil {
-			return common.Hash{}, err
-		} else {
-			return common.BytesToHash(crypto.Keccak256(logListRlp)), nil
-		}
-	}
-}
-
-func (api *RetestethAPI) BlockNumber(ctx context.Context) (uint64, error) {
-	return api.currentNumber(), nil
-}
-
-func (api *RetestethAPI) GetBlockByNumber(ctx context.Context, blockNr math.HexOrDecimal64, fullTx bool) (map[string]interface{}, error) {
-	block := api.blockchain.GetBlockByNumber(uint64(blockNr))
-	if block != nil {
-		response, err := RPCMarshalBlock(block, true, fullTx)
-		if err != nil {
-			return nil, err
-		}
-		response["author"] = response["miner"]
-		response["totalDifficulty"] = (*hexutil.Big)(api.blockchain.GetTd(block.Hash(), uint64(blockNr)))
-		return response, err
-	}
-	return nil, fmt.Errorf("block %d not found", blockNr)
-}
-
-func (api *RetestethAPI) GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error) {
-	block := api.blockchain.GetBlockByHash(blockHash)
-	if block != nil {
-		response, err := RPCMarshalBlock(block, true, fullTx)
-		if err != nil {
-			return nil, err
-		}
-		response["author"] = response["miner"]
-		response["totalDifficulty"] = (*hexutil.Big)(api.blockchain.GetTd(block.Hash(), block.Number().Uint64()))
-		return response, err
-	}
-	return nil, fmt.Errorf("block 0x%x not found", blockHash)
-}
-
-func (api *RetestethAPI) AccountRange(ctx context.Context,
-	blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
-	addressHash *math.HexOrDecimal256, maxResults uint64,
-) (AccountRangeResult, error) {
-	var (
-		header *types.Header
-		block  *types.Block
-	)
-	if (*big.Int)(blockHashOrNumber).Cmp(big.NewInt(math.MaxInt64)) > 0 {
-		blockHash := common.BigToHash((*big.Int)(blockHashOrNumber))
-		header = api.blockchain.GetHeaderByHash(blockHash)
-		block = api.blockchain.GetBlockByHash(blockHash)
-		//fmt.Printf("Account range: %x, txIndex %d, start: %x, maxResults: %d\n", blockHash, txIndex, common.BigToHash((*big.Int)(addressHash)), maxResults)
-	} else {
-		blockNumber := (*big.Int)(blockHashOrNumber).Uint64()
-		header = api.blockchain.GetHeaderByNumber(blockNumber)
-		block = api.blockchain.GetBlockByNumber(blockNumber)
-		//fmt.Printf("Account range: %d, txIndex %d, start: %x, maxResults: %d\n", blockNumber, txIndex, common.BigToHash((*big.Int)(addressHash)), maxResults)
-	}
-	parentHeader := api.blockchain.GetHeaderByHash(header.ParentHash)
-	var root common.Hash
-	var statedb *state.StateDB
-	var err error
-	if parentHeader == nil || int(txIndex) >= len(block.Transactions()) {
-		root = header.Root
-		statedb, err = api.blockchain.StateAt(root)
-		if err != nil {
-			return AccountRangeResult{}, err
-		}
-	} else {
-		root = parentHeader.Root
-		statedb, err = api.blockchain.StateAt(root)
-		if err != nil {
-			return AccountRangeResult{}, err
-		}
-		// Recompute transactions up to the target index.
-		signer := types.MakeSigner(api.blockchain.Config(), block.Number())
-		for idx, tx := range block.Transactions() {
-			// Assemble the transaction call message and return if the requested offset
-			msg, _ := tx.AsMessage(signer)
-			context := core.NewEVMContext(msg, block.Header(), api.blockchain, nil)
-			// Not yet the searched for transaction, execute on top of the current state
-			vmenv := vm.NewEVM(context, statedb, api.blockchain.Config(), vm.Config{})
-			if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil {
-				return AccountRangeResult{}, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err)
-			}
-			// Ensure any modifications are committed to the state
-			// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
-			root = statedb.IntermediateRoot(vmenv.ChainConfig().IsEIP158(block.Number()))
-			if idx == int(txIndex) {
-				// This is to make sure root can be opened by OpenTrie
-				root, err = statedb.Commit(api.chainConfig.IsEIP158(block.Number()))
-				if err != nil {
-					return AccountRangeResult{}, err
-				}
-				break
-			}
-		}
-	}
-	accountTrie, err := statedb.Database().OpenTrie(root)
-	if err != nil {
-		return AccountRangeResult{}, err
-	}
-	it := trie.NewIterator(accountTrie.NodeIterator(common.BigToHash((*big.Int)(addressHash)).Bytes()))
-	result := AccountRangeResult{AddressMap: make(map[common.Hash]common.Address)}
-	for i := 0; i < int(maxResults) && it.Next(); i++ {
-		if preimage := accountTrie.GetKey(it.Key); preimage != nil {
-			result.AddressMap[common.BytesToHash(it.Key)] = common.BytesToAddress(preimage)
-		}
-	}
-	//fmt.Printf("Number of entries returned: %d\n", len(result.AddressMap))
-	// Add the 'next key' so clients can continue downloading.
-	if it.Next() {
-		next := common.BytesToHash(it.Key)
-		result.NextKey = next
-	}
-	return result, nil
-}
-
-func (api *RetestethAPI) GetBalance(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (*math.HexOrDecimal256, error) {
-	//fmt.Printf("GetBalance %x, block %d\n", address, blockNr)
-	header := api.blockchain.GetHeaderByNumber(uint64(blockNr))
-	statedb, err := api.blockchain.StateAt(header.Root)
-	if err != nil {
-		return nil, err
-	}
-	return (*math.HexOrDecimal256)(statedb.GetBalance(address)), nil
-}
-
-func (api *RetestethAPI) GetCode(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (hexutil.Bytes, error) {
-	header := api.blockchain.GetHeaderByNumber(uint64(blockNr))
-	statedb, err := api.blockchain.StateAt(header.Root)
-	if err != nil {
-		return nil, err
-	}
-	return statedb.GetCode(address), nil
-}
-
-func (api *RetestethAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (uint64, error) {
-	header := api.blockchain.GetHeaderByNumber(uint64(blockNr))
-	statedb, err := api.blockchain.StateAt(header.Root)
-	if err != nil {
-		return 0, err
-	}
-	return statedb.GetNonce(address), nil
-}
-
-func (api *RetestethAPI) StorageRangeAt(ctx context.Context,
-	blockHashOrNumber *math.HexOrDecimal256, txIndex uint64,
-	address common.Address,
-	begin *math.HexOrDecimal256, maxResults uint64,
-) (StorageRangeResult, error) {
-	var (
-		header *types.Header
-		block  *types.Block
-	)
-	if (*big.Int)(blockHashOrNumber).Cmp(big.NewInt(math.MaxInt64)) > 0 {
-		blockHash := common.BigToHash((*big.Int)(blockHashOrNumber))
-		header = api.blockchain.GetHeaderByHash(blockHash)
-		block = api.blockchain.GetBlockByHash(blockHash)
-		//fmt.Printf("Storage range: %x, txIndex %d, addr: %x, start: %x, maxResults: %d\n",
-		//	blockHash, txIndex, address, common.BigToHash((*big.Int)(begin)), maxResults)
-	} else {
-		blockNumber := (*big.Int)(blockHashOrNumber).Uint64()
-		header = api.blockchain.GetHeaderByNumber(blockNumber)
-		block = api.blockchain.GetBlockByNumber(blockNumber)
-		//fmt.Printf("Storage range: %d, txIndex %d, addr: %x, start: %x, maxResults: %d\n",
-		//	blockNumber, txIndex, address, common.BigToHash((*big.Int)(begin)), maxResults)
-	}
-	parentHeader := api.blockchain.GetHeaderByHash(header.ParentHash)
-	var root common.Hash
-	var statedb *state.StateDB
-	var err error
-	if parentHeader == nil || int(txIndex) >= len(block.Transactions()) {
-		root = header.Root
-		statedb, err = api.blockchain.StateAt(root)
-		if err != nil {
-			return StorageRangeResult{}, err
-		}
-	} else {
-		root = parentHeader.Root
-		statedb, err = api.blockchain.StateAt(root)
-		if err != nil {
-			return StorageRangeResult{}, err
-		}
-		// Recompute transactions up to the target index.
-		signer := types.MakeSigner(api.blockchain.Config(), block.Number())
-		for idx, tx := range block.Transactions() {
-			// Assemble the transaction call message and return if the requested offset
-			msg, _ := tx.AsMessage(signer)
-			context := core.NewEVMContext(msg, block.Header(), api.blockchain, nil)
-			// Not yet the searched for transaction, execute on top of the current state
-			vmenv := vm.NewEVM(context, statedb, api.blockchain.Config(), vm.Config{})
-			if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil {
-				return StorageRangeResult{}, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err)
-			}
-			// Ensure any modifications are committed to the state
-			// Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect
-			_ = statedb.IntermediateRoot(vmenv.ChainConfig().IsEIP158(block.Number()))
-			if idx == int(txIndex) {
-				// This is to make sure root can be opened by OpenTrie
-				_, err = statedb.Commit(vmenv.ChainConfig().IsEIP158(block.Number()))
-				if err != nil {
-					return StorageRangeResult{}, err
-				}
-			}
-		}
-	}
-	storageTrie := statedb.StorageTrie(address)
-	it := trie.NewIterator(storageTrie.NodeIterator(common.BigToHash((*big.Int)(begin)).Bytes()))
-	result := StorageRangeResult{Storage: make(map[common.Hash]SRItem)}
-	for i := 0; /*i < int(maxResults) && */ it.Next(); i++ {
-		if preimage := storageTrie.GetKey(it.Key); preimage != nil {
-			key := (*math.HexOrDecimal256)(big.NewInt(0).SetBytes(preimage))
-			v, _, err := rlp.SplitString(it.Value)
-			if err != nil {
-				return StorageRangeResult{}, err
-			}
-			value := (*math.HexOrDecimal256)(big.NewInt(0).SetBytes(v))
-			ks, _ := key.MarshalText()
-			vs, _ := value.MarshalText()
-			if len(ks)%2 != 0 {
-				ks = append(append(append([]byte{}, ks[:2]...), byte('0')), ks[2:]...)
-			}
-			if len(vs)%2 != 0 {
-				vs = append(append(append([]byte{}, vs[:2]...), byte('0')), vs[2:]...)
-			}
-			result.Storage[common.BytesToHash(it.Key)] = SRItem{
-				Key:   string(ks),
-				Value: string(vs),
-			}
-		}
-	}
-	if it.Next() {
-		result.Complete = false
-	} else {
-		result.Complete = true
-	}
-	return result, nil
-}
-
-func (api *RetestethAPI) ClientVersion(ctx context.Context) (string, error) {
-	return "Geth-" + params.VersionWithCommit(gitCommit, gitDate), nil
-}
-
-func retesteth(ctx *cli.Context) error {
-	log.Info("Welcome to retesteth!")
-	// register signer API with server
-	var (
-		extapiURL string
-	)
-	apiImpl := &RetestethAPI{}
-	var testApi RetestethTestAPI = apiImpl
-	var ethApi RetestethEthAPI = apiImpl
-	var debugApi RetestethDebugAPI = apiImpl
-	var web3Api RetestWeb3API = apiImpl
-	rpcAPI := []rpc.API{
-		{
-			Namespace: "test",
-			Public:    true,
-			Service:   testApi,
-			Version:   "1.0",
-		},
-		{
-			Namespace: "eth",
-			Public:    true,
-			Service:   ethApi,
-			Version:   "1.0",
-		},
-		{
-			Namespace: "debug",
-			Public:    true,
-			Service:   debugApi,
-			Version:   "1.0",
-		},
-		{
-			Namespace: "web3",
-			Public:    true,
-			Service:   web3Api,
-			Version:   "1.0",
-		},
-	}
-	vhosts := utils.SplitAndTrim(ctx.GlobalString(utils.HTTPVirtualHostsFlag.Name))
-	cors := utils.SplitAndTrim(ctx.GlobalString(utils.HTTPCORSDomainFlag.Name))
-
-	// register apis and create handler stack
-	srv := rpc.NewServer()
-	err := node.RegisterApisFromWhitelist(rpcAPI, []string{"test", "eth", "debug", "web3"}, srv, false)
-	if err != nil {
-		utils.Fatalf("Could not register RPC apis: %w", err)
-	}
-	handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
-
-	// start http server
-	var RetestethHTTPTimeouts = rpc.HTTPTimeouts{
-		ReadTimeout:  120 * time.Second,
-		WriteTimeout: 120 * time.Second,
-		IdleTimeout:  120 * time.Second,
-	}
-	httpEndpoint := fmt.Sprintf("%s:%d", ctx.GlobalString(utils.HTTPListenAddrFlag.Name), ctx.Int(rpcPortFlag.Name))
-	httpServer, _, err := node.StartHTTPEndpoint(httpEndpoint, RetestethHTTPTimeouts, handler)
-	if err != nil {
-		utils.Fatalf("Could not start RPC api: %v", err)
-	}
-	extapiURL = fmt.Sprintf("http://%s", httpEndpoint)
-	log.Info("HTTP endpoint opened", "url", extapiURL)
-
-	defer func() {
-		// Don't bother imposing a timeout here.
-		httpServer.Shutdown(context.Background())
-		log.Info("HTTP endpoint closed", "url", httpEndpoint)
-	}()
-
-	abortChan := make(chan os.Signal, 11)
-	signal.Notify(abortChan, os.Interrupt)
-
-	sig := <-abortChan
-	log.Info("Exiting...", "signal", sig)
-	return nil
-}

cmd/geth/retesteth_copypaste.go

@@ -1,148 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
-type RPCTransaction struct {
-	BlockHash        common.Hash     `json:"blockHash"`
-	BlockNumber      *hexutil.Big    `json:"blockNumber"`
-	From             common.Address  `json:"from"`
-	Gas              hexutil.Uint64  `json:"gas"`
-	GasPrice         *hexutil.Big    `json:"gasPrice"`
-	Hash             common.Hash     `json:"hash"`
-	Input            hexutil.Bytes   `json:"input"`
-	Nonce            hexutil.Uint64  `json:"nonce"`
-	To               *common.Address `json:"to"`
-	TransactionIndex hexutil.Uint    `json:"transactionIndex"`
-	Value            *hexutil.Big    `json:"value"`
-	V                *hexutil.Big    `json:"v"`
-	R                *hexutil.Big    `json:"r"`
-	S                *hexutil.Big    `json:"s"`
-}
-
-// newRPCTransaction returns a transaction that will serialize to the RPC
-// representation, with the given location metadata set (if available).
-func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
-	var signer types.Signer = types.FrontierSigner{}
-	if tx.Protected() {
-		signer = types.NewEIP155Signer(tx.ChainId())
-	}
-	from, _ := types.Sender(signer, tx)
-	v, r, s := tx.RawSignatureValues()
-
-	result := &RPCTransaction{
-		From:     from,
-		Gas:      hexutil.Uint64(tx.Gas()),
-		GasPrice: (*hexutil.Big)(tx.GasPrice()),
-		Hash:     tx.Hash(),
-		Input:    hexutil.Bytes(tx.Data()),
-		Nonce:    hexutil.Uint64(tx.Nonce()),
-		To:       tx.To(),
-		Value:    (*hexutil.Big)(tx.Value()),
-		V:        (*hexutil.Big)(v),
-		R:        (*hexutil.Big)(r),
-		S:        (*hexutil.Big)(s),
-	}
-	if blockHash != (common.Hash{}) {
-		result.BlockHash = blockHash
-		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
-		result.TransactionIndex = hexutil.Uint(index)
-	}
-	return result
-}
-
-// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
-func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
-	txs := b.Transactions()
-	if index >= uint64(len(txs)) {
-		return nil
-	}
-	return newRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
-}
-
-// newRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
-func newRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
-	for idx, tx := range b.Transactions() {
-		if tx.Hash() == hash {
-			return newRPCTransactionFromBlockIndex(b, uint64(idx))
-		}
-	}
-	return nil
-}
-
-// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
-// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
-// transaction hashes.
-func RPCMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	head := b.Header() // copies the header once
-	fields := map[string]interface{}{
-		"number":           (*hexutil.Big)(head.Number),
-		"hash":             b.Hash(),
-		"parentHash":       head.ParentHash,
-		"nonce":            head.Nonce,
-		"mixHash":          head.MixDigest,
-		"sha3Uncles":       head.UncleHash,
-		"logsBloom":        head.Bloom,
-		"stateRoot":        head.Root,
-		"miner":            head.Coinbase,
-		"difficulty":       (*hexutil.Big)(head.Difficulty),
-		"extraData":        hexutil.Bytes(head.Extra),
-		"size":             hexutil.Uint64(b.Size()),
-		"gasLimit":         hexutil.Uint64(head.GasLimit),
-		"gasUsed":          hexutil.Uint64(head.GasUsed),
-		"timestamp":        hexutil.Uint64(head.Time),
-		"transactionsRoot": head.TxHash,
-		"receiptsRoot":     head.ReceiptHash,
-	}
-
-	if inclTx {
-		formatTx := func(tx *types.Transaction) (interface{}, error) {
-			return tx.Hash(), nil
-		}
-		if fullTx {
-			formatTx = func(tx *types.Transaction) (interface{}, error) {
-				return newRPCTransactionFromBlockHash(b, tx.Hash()), nil
-			}
-		}
-		txs := b.Transactions()
-		transactions := make([]interface{}, len(txs))
-		var err error
-		for i, tx := range txs {
-			if transactions[i], err = formatTx(tx); err != nil {
-				return nil, err
-			}
-		}
-		fields["transactions"] = transactions
-	}
-
-	uncles := b.Uncles()
-	uncleHashes := make([]common.Hash, len(uncles))
-	for i, uncle := range uncles {
-		uncleHashes[i] = uncle.Hash()
-	}
-	fields["uncles"] = uncleHashes
-
-	return fields, nil
-}

cmd/geth/run_test.go

@@ -70,12 +70,12 @@ func runGeth(t *testing.T, args ...string) *testgeth {
 	tt := &testgeth{}
 	tt.TestCmd = cmdtest.NewTestCmd(t, tt)
 	for i, arg := range args {
-		switch {
-		case arg == "-datadir" || arg == "--datadir":
+		switch arg {
+		case "--datadir":
 			if i < len(args)-1 {
 				tt.Datadir = args[i+1]
 			}
-		case arg == "-etherbase" || arg == "--etherbase":
+		case "--miner.etherbase":
 			if i < len(args)-1 {
 				tt.Etherbase = args[i+1]
 			}
@@ -84,7 +84,7 @@ func runGeth(t *testing.T, args ...string) *testgeth {
 	if tt.Datadir == "" {
 		tt.Datadir = tmpdir(t)
 		tt.Cleanup = func() { os.RemoveAll(tt.Datadir) }
-		args = append([]string{"-datadir", tt.Datadir}, args...)
+		args = append([]string{"--datadir", tt.Datadir}, args...)
 		// Remove the temporary datadir if something fails below.
 		defer func() {
 			if t.Failed() {

cmd/geth/snapshot.go

@@ -0,0 +1,433 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"time"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/state/pruner"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var (
+	// emptyRoot is the known root hash of an empty trie.
+	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+
+	// emptyCode is the known hash of the empty EVM bytecode.
+	emptyCode = crypto.Keccak256(nil)
+)
+
+var (
+	snapshotCommand = cli.Command{
+		Name:        "snapshot",
+		Usage:       "A set of commands based on the snapshot",
+		Category:    "MISCELLANEOUS COMMANDS",
+		Description: "",
+		Subcommands: []cli.Command{
+			{
+				Name:      "prune-state",
+				Usage:     "Prune stale ethereum state data based on the snapshot",
+				ArgsUsage: "<root>",
+				Action:    utils.MigrateFlags(pruneState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+					utils.CacheTrieJournalFlag,
+					utils.BloomFilterSizeFlag,
+				},
+				Description: `
+geth snapshot prune-state <state-root>
+will prune historical state data with the help of the state snapshot.
+All trie nodes and contract codes that do not belong to the specified
+version state will be deleted from the database. After pruning, only
+two version states are available: genesis and the specific one.
+
+The default pruning target is the HEAD-127 state.
+
+WARNING: It's necessary to delete the trie clean cache after the pruning.
+If you specify another directory for the trie clean cache via "--cache.trie.journal"
+during the use of Geth, please also specify it here for correct deletion. Otherwise
+the trie clean cache with default directory will be deleted.
+`,
+			},
+			{
+				Name:      "verify-state",
+				Usage:     "Recalculate state hash based on the snapshot for verification",
+				ArgsUsage: "<root>",
+				Action:    utils.MigrateFlags(verifyState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
+geth snapshot verify-state <state-root>
+will traverse the whole accounts and storages set based on the specified
+snapshot and recalculate the root hash of state for verification.
+In other words, this command does the snapshot to trie conversion.
+`,
+			},
+			{
+				Name:      "traverse-state",
+				Usage:     "Traverse the state with given root hash for verification",
+				ArgsUsage: "<root>",
+				Action:    utils.MigrateFlags(traverseState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
+geth snapshot traverse-state <state-root>
+will traverse the whole state from the given state root and will abort if any
+referenced trie node or contract code is missing. This command can be used for
+state integrity verification. The default checking target is the HEAD state.
+
+It's also usable without snapshot enabled.
+`,
+			},
+			{
+				Name:      "traverse-rawstate",
+				Usage:     "Traverse the state with given root hash for verification",
+				ArgsUsage: "<root>",
+				Action:    utils.MigrateFlags(traverseRawState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
+geth snapshot traverse-rawstate <state-root>
+will traverse the whole state from the given root and will abort if any referenced
+trie node or contract code is missing. This command can be used for state integrity
+verification. The default checking target is the HEAD state. It's basically identical
+to traverse-state, but the check granularity is smaller. 
+
+It's also usable without snapshot enabled.
+`,
+			},
+		},
+	}
+)
+
+func pruneState(ctx *cli.Context) error {
+	stack, config := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chain, chaindb := utils.MakeChain(ctx, stack, true)
+	defer chaindb.Close()
+
+	pruner, err := pruner.NewPruner(chaindb, chain.CurrentBlock().Header(), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name))
+	if err != nil {
+		log.Error("Failed to open snapshot tree", "error", err)
+		return err
+	}
+	if ctx.NArg() > 1 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+	var targetRoot common.Hash
+	if ctx.NArg() == 1 {
+		targetRoot, err = parseRoot(ctx.Args()[0])
+		if err != nil {
+			log.Error("Failed to resolve state root", "error", err)
+			return err
+		}
+	}
+	if err = pruner.Prune(targetRoot); err != nil {
+		log.Error("Failed to prune state", "error", err)
+		return err
+	}
+	return nil
+}
+
+func verifyState(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chain, chaindb := utils.MakeChain(ctx, stack, true)
+	defer chaindb.Close()
+
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, chain.CurrentBlock().Root(), false, false, false)
+	if err != nil {
+		log.Error("Failed to open snapshot tree", "error", err)
+		return err
+	}
+	if ctx.NArg() > 1 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+	var root = chain.CurrentBlock().Root()
+	if ctx.NArg() == 1 {
+		root, err = parseRoot(ctx.Args()[0])
+		if err != nil {
+			log.Error("Failed to resolve state root", "error", err)
+			return err
+		}
+	}
+	if err := snaptree.Verify(root); err != nil {
+		log.Error("Failed to verfiy state", "error", err)
+		return err
+	}
+	log.Info("Verified the state")
+	return nil
+}
+
+// traverseState is a helper function used for pruning verification.
+// Basically it just iterates the trie, ensure all nodes and associated
+// contract codes are present.
+func traverseState(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chain, chaindb := utils.MakeChain(ctx, stack, true)
+	defer chaindb.Close()
+
+	if ctx.NArg() > 1 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+	// Use the HEAD root as the default
+	head := chain.CurrentBlock()
+	if head == nil {
+		log.Error("Head block is missing")
+		return errors.New("head block is missing")
+	}
+	var (
+		root common.Hash
+		err  error
+	)
+	if ctx.NArg() == 1 {
+		root, err = parseRoot(ctx.Args()[0])
+		if err != nil {
+			log.Error("Failed to resolve state root", "error", err)
+			return err
+		}
+		log.Info("Start traversing the state", "root", root)
+	} else {
+		root = head.Root()
+		log.Info("Start traversing the state", "root", root, "number", head.NumberU64())
+	}
+	triedb := trie.NewDatabase(chaindb)
+	t, err := trie.NewSecure(root, triedb)
+	if err != nil {
+		log.Error("Failed to open trie", "root", root, "error", err)
+		return err
+	}
+	var (
+		accounts   int
+		slots      int
+		codes      int
+		lastReport time.Time
+		start      = time.Now()
+	)
+	accIter := trie.NewIterator(t.NodeIterator(nil))
+	for accIter.Next() {
+		accounts += 1
+		var acc state.Account
+		if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil {
+			log.Error("Invalid account encountered during traversal", "error", err)
+			return err
+		}
+		if acc.Root != emptyRoot {
+			storageTrie, err := trie.NewSecure(acc.Root, triedb)
+			if err != nil {
+				log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
+				return err
+			}
+			storageIter := trie.NewIterator(storageTrie.NodeIterator(nil))
+			for storageIter.Next() {
+				slots += 1
+			}
+			if storageIter.Err != nil {
+				log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err)
+				return storageIter.Err
+			}
+		}
+		if !bytes.Equal(acc.CodeHash, emptyCode) {
+			code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash))
+			if len(code) == 0 {
+				log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash))
+				return errors.New("missing code")
+			}
+			codes += 1
+		}
+		if time.Since(lastReport) > time.Second*8 {
+			log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+			lastReport = time.Now()
+		}
+	}
+	if accIter.Err != nil {
+		log.Error("Failed to traverse state trie", "root", root, "error", accIter.Err)
+		return accIter.Err
+	}
+	log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+	return nil
+}
+
+// traverseRawState is a helper function used for pruning verification.
+// Basically it just iterates the trie, ensure all nodes and associated
+// contract codes are present. It's basically identical to traverseState
+// but it will check each trie node.
+func traverseRawState(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chain, chaindb := utils.MakeChain(ctx, stack, true)
+	defer chaindb.Close()
+
+	if ctx.NArg() > 1 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+	// Use the HEAD root as the default
+	head := chain.CurrentBlock()
+	if head == nil {
+		log.Error("Head block is missing")
+		return errors.New("head block is missing")
+	}
+	var (
+		root common.Hash
+		err  error
+	)
+	if ctx.NArg() == 1 {
+		root, err = parseRoot(ctx.Args()[0])
+		if err != nil {
+			log.Error("Failed to resolve state root", "error", err)
+			return err
+		}
+		log.Info("Start traversing the state", "root", root)
+	} else {
+		root = head.Root()
+		log.Info("Start traversing the state", "root", root, "number", head.NumberU64())
+	}
+	triedb := trie.NewDatabase(chaindb)
+	t, err := trie.NewSecure(root, triedb)
+	if err != nil {
+		log.Error("Failed to open trie", "root", root, "error", err)
+		return err
+	}
+	var (
+		nodes      int
+		accounts   int
+		slots      int
+		codes      int
+		lastReport time.Time
+		start      = time.Now()
+	)
+	accIter := t.NodeIterator(nil)
+	for accIter.Next(true) {
+		nodes += 1
+		node := accIter.Hash()
+
+		if node != (common.Hash{}) {
+			// Check the present for non-empty hash node(embedded node doesn't
+			// have their own hash).
+			blob := rawdb.ReadTrieNode(chaindb, node)
+			if len(blob) == 0 {
+				log.Error("Missing trie node(account)", "hash", node)
+				return errors.New("missing account")
+			}
+		}
+		// If it's a leaf node, yes we are touching an account,
+		// dig into the storage trie further.
+		if accIter.Leaf() {
+			accounts += 1
+			var acc state.Account
+			if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil {
+				log.Error("Invalid account encountered during traversal", "error", err)
+				return errors.New("invalid account")
+			}
+			if acc.Root != emptyRoot {
+				storageTrie, err := trie.NewSecure(acc.Root, triedb)
+				if err != nil {
+					log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
+					return errors.New("missing storage trie")
+				}
+				storageIter := storageTrie.NodeIterator(nil)
+				for storageIter.Next(true) {
+					nodes += 1
+					node := storageIter.Hash()
+
+					// Check the present for non-empty hash node(embedded node doesn't
+					// have their own hash).
+					if node != (common.Hash{}) {
+						blob := rawdb.ReadTrieNode(chaindb, node)
+						if len(blob) == 0 {
+							log.Error("Missing trie node(storage)", "hash", node)
+							return errors.New("missing storage")
+						}
+					}
+					// Bump the counter if it's leaf node.
+					if storageIter.Leaf() {
+						slots += 1
+					}
+				}
+				if storageIter.Error() != nil {
+					log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error())
+					return storageIter.Error()
+				}
+			}
+			if !bytes.Equal(acc.CodeHash, emptyCode) {
+				code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash))
+				if len(code) == 0 {
+					log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey()))
+					return errors.New("missing code")
+				}
+				codes += 1
+			}
+			if time.Since(lastReport) > time.Second*8 {
+				log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+				lastReport = time.Now()
+			}
+		}
+	}
+	if accIter.Error() != nil {
+		log.Error("Failed to traverse state trie", "root", root, "error", accIter.Error())
+		return accIter.Error()
+	}
+	log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+	return nil
+}
+
+func parseRoot(input string) (common.Hash, error) {
+	var h common.Hash
+	if err := h.UnmarshalText([]byte(input)); err != nil {
+		return h, err
+	}
+	return h, nil
+}

cmd/geth/testdata/vcheck/data.json

@@ -0,0 +1,61 @@
+[
+  {
+    "name": "CorruptedDAG",
+    "uid": "GETH-2020-01",
+    "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.",
+    "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.",
+    "links": [
+      "https://github.com/ethereum/go-ethereum/pull/21793",
+      "https://blog.ethereum.org/2020/11/12/geth_security_release/",
+      "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49"
+    ],
+    "introduced": "v1.6.0",
+    "fixed": "v1.9.24",
+    "published": "2020-11-12",
+    "severity": "Medium",
+    "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.2(1|2|3)-.*"
+  },
+  {
+    "name": "GoCrash",
+    "uid": "GETH-2020-02",
+    "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`",
+    "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.",
+    "links": [
+      "https://blog.ethereum.org/2020/11/12/geth_security_release/",
+      "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM",
+      "https://github.com/golang/go/issues/42552"
+    ],
+    "fixed": "v1.9.24",
+    "published": "2020-11-12",
+    "severity": "Critical",
+    "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$"
+  },
+  {
+    "name": "ShallowCopy",
+    "uid": "GETH-2020-03",
+    "summary": "A consensus flaw in Geth, related to `datacopy` precompile",
+    "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.",
+    "links": [
+      "https://blog.ethereum.org/2020/11/12/geth_security_release/"
+    ],
+    "introduced": "v1.9.7",
+    "fixed": "v1.9.17",
+    "published": "2020-11-12",
+    "severity": "Critical",
+    "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$"
+  },
+  {
+    "name": "GethCrash",
+    "uid": "GETH-2020-04",
+    "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing",
+    "description": "Full details to be disclosed at a later date",
+    "links": [
+      "https://blog.ethereum.org/2020/11/12/geth_security_release/"
+    ],
+    "introduced": "v1.9.16",
+    "fixed": "v1.9.18",
+    "published": "2020-11-12",
+    "severity": "Critical",
+    "check": "Geth\\/v1\\.9.(16|17).*$"
+  }
+]

cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1

@@ -0,0 +1,4 @@
+untrusted comment: signature from minisign secret key
+RWQkliYstQBOKFQFQTjmCd6TPw07VZyWFSB3v4+1BM1kv8eHLE5FDy2OkPEqtdaL53xftlrHoJQie0uCcovdlSV8kpyxiLrxEQ0=
+trusted comment: timestamp:1605618622	file:vulnerabilities.json
+osAPs4QPdDkmiWQxqeMIzYv/b+ZGxJ+19Sbrk1Cpq4t2gHBT+lqFtwL3OCzKWWyjGRTmHfsVGBYpzEdPRQ0/BQ==

cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2

@@ -0,0 +1,4 @@
+untrusted comment: Here's a comment
+RWQkliYstQBOKFQFQTjmCd6TPw07VZyWFSB3v4+1BM1kv8eHLE5FDy2OkPEqtdaL53xftlrHoJQie0uCcovdlSV8kpyxiLrxEQ0=
+trusted comment: Here's a trusted comment
+3CnkIuz9MEDa7uNyGZAbKZhuirwfiqm7E1uQHrd2SiO4Y8+Akw9vs052AyKw0s5nhbYHCZE2IMQdHNjKwxEGAQ==

cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3

@@ -0,0 +1,4 @@
+untrusted comment: One more (untrusted) comment
+RWQkliYstQBOKFQFQTjmCd6TPw07VZyWFSB3v4+1BM1kv8eHLE5FDy2OkPEqtdaL53xftlrHoJQie0uCcovdlSV8kpyxiLrxEQ0=
+trusted comment: Here's a trusted comment
+3CnkIuz9MEDa7uNyGZAbKZhuirwfiqm7E1uQHrd2SiO4Y8+Akw9vs052AyKw0s5nhbYHCZE2IMQdHNjKwxEGAQ==

cmd/geth/testdata/vcheck/minisign.pub

@@ -0,0 +1,2 @@
+untrusted comment: minisign public key 284E00B52C269624
+RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp

cmd/geth/testdata/vcheck/minisign.sec

@@ -0,0 +1,2 @@
+untrusted comment: minisign encrypted secret key
+RWRTY0Iyz8kmPMKrqk6DCtlO9a33akKiaOQG1aLolqDxs52qvPoAAAACAAAAAAAAAEAAAAAArEiggdvyn6+WzTprirLtgiYQoU+ihz/HyGgjhuF+Pz2ddMduyCO+xjCHeq+vgVVW039fbsI8hW6LRGJZLBKV5/jdxCXAVVQE7qTQ6xpEdO0z8Z731/pV1hlspQXG2PNd16NMtwd9dWw=

cmd/geth/testdata/vcheck/signify-sigs/data.json.sig

@@ -0,0 +1,2 @@
+untrusted comment: verify with ./signifykey.pub
+RWSKLNhZb0KdAbhRUhW2LQZXdnwttu2SYhM9EuC4mMgOJB85h7/YIPupf8/ldTs4N8e9Y/fhgdY40q5LQpt5IFC62fq0v8U1/w8=

cmd/geth/testdata/vcheck/signifykey.pub

@@ -0,0 +1,2 @@
+untrusted comment: signify public key
+RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/