Merge pull request #3075 from ethereum/release/1.4

Geth 1.4.15: Come at me Bro

.travis.yml

@@ -27,7 +27,7 @@ matrix:
             - debhelper
             - dput
       script:
-        - go run build/ci.go travis-debsrc
+        - go run build/ci.go debsrc -signer "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>" -upload ppa:lp-fjl/geth-ci-testing
 
 install:
   - go get golang.org/x/tools/cmd/cover

Makefile

@@ -42,12 +42,12 @@ geth-linux: geth-linux-386 geth-linux-amd64 geth-linux-arm geth-linux-mips64 get
 	@ls -ld $(GOBIN)/geth-linux-*
 
 geth-linux-386:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/386 -v ./cmd/geth
 	@echo "Linux 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep 386
 
 geth-linux-amd64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/amd64 -v ./cmd/geth
 	@echo "Linux amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep amd64
 
@@ -56,32 +56,32 @@ geth-linux-arm: geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-ar
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm
 
 geth-linux-arm-5:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-5 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-5 -v ./cmd/geth
 	@echo "Linux ARMv5 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-5
 
 geth-linux-arm-6:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-6 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-6 -v ./cmd/geth
 	@echo "Linux ARMv6 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-6
 
 geth-linux-arm-7:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-7 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-7 -v ./cmd/geth
 	@echo "Linux ARMv7 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-7
 
 geth-linux-arm64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/arm64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm64 -v ./cmd/geth
 	@echo "Linux ARM64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm64
 
 geth-linux-mips64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64 -v ./cmd/geth
 	@echo "Linux MIPS64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64
 
 geth-linux-mips64le:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64le -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64le -v ./cmd/geth
 	@echo "Linux MIPS64le cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64le
 
@@ -90,12 +90,12 @@ geth-darwin: geth-darwin-386 geth-darwin-amd64
 	@ls -ld $(GOBIN)/geth-darwin-*
 
 geth-darwin-386:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=darwin/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/386 -v ./cmd/geth
 	@echo "Darwin 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep 386
 
 geth-darwin-amd64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=darwin/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/amd64 -v ./cmd/geth
 	@echo "Darwin amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep amd64
 
@@ -104,21 +104,21 @@ geth-windows: geth-windows-386 geth-windows-amd64
 	@ls -ld $(GOBIN)/geth-windows-*
 
 geth-windows-386:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=windows/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/386 -v ./cmd/geth
 	@echo "Windows 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep 386
 
 geth-windows-amd64:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=windows/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/amd64 -v ./cmd/geth
 	@echo "Windows amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep amd64
 
 geth-android:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=android-21/aar -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=android-21/aar -v ./cmd/geth
 	@echo "Android cross compilation done:"
 	@ls -ld $(GOBIN)/geth-android-*
 
 geth-ios:
-	build/env.sh go run build/ci.go xgo --go=$(GO) --dest=$(GOBIN) --targets=ios-7.0/framework -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=ios-7.0/framework -v ./cmd/geth
 	@echo "iOS framework cross compilation done:"
 	@ls -ld $(GOBIN)/geth-ios-*

VERSION

@@ -1 +1 @@
-1.4.12
+1.4.15

appveyor.yml

@@ -6,26 +6,28 @@ clone_depth: 5
 version: "{branch}.{build}"
 environment:
   global:
+    # Go stuff
     GOPATH: c:\gopath
-
+    GO: c:\go\bin\go
-# cache choco package files so we don't hit sourceforge all
+    GOROOT: c:\go
-# the time.
+    CC: C:\msys64\mingw64\bin\gcc.exe
-cache:
+    # MSYS2 stuff
-  - c:\cache
+    MSYS2_ARCH: x86_64
+    MSYSTEM: MINGW64
+    PATH: C:\msys64\mingw64\bin\;%PATH%
 
 install:
-  - cmd: choco install --cache c:\cache golang mingw | find /v "Extracting  "
+  - "%GO% version"
-  - refreshenv
+  - "%CC% --version"
-  - cd c:\gopath\src\github.com\ethereum\go-ethereum
 
 build_script:
-  - go run build\ci.go install
+  - "%GO% run build\\ci.go install"
 
 test_script:
-  - go run build\ci.go test -vet -coverage
+  - "%GO% run build\\ci.go test -vet -coverage"
 
 after_build:
-  - go run build\ci.go archive -type zip
+  - "%GO% run build\\ci.go archive -type zip"
 
 artifacts:
   - path: geth-*.zip

build/ci-notes.md

@@ -1,5 +1,4 @@
-Debian Packaging
+# Debian Packaging
-----------------
 
 Tagged releases and develop branch commits are available as installable Debian packages
 for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
@@ -8,6 +7,7 @@ Canonical:
 - Trusty Tahr (14.04 LTS)
 - Wily Werewolf (15.10)
 - Xenial Xerus (16.04 LTS)
+- Yakkety Yak (16.10)
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -21,6 +21,29 @@ variable which Travis CI makes available to certain builds.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.6, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.7, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Elp-fjl/+archive/ubuntu/geth-ci-testing/+edit-dependencies
 
+## Building Packages Locally (for testing)
+
+You need to run Ubuntu to do test packaging.
+
+Add the gophers PPA and install Go 1.7 and Debian packaging tools:
+
+    $ sudo apt-add-repository ppa:gophers/ubuntu/archive
+    $ sudo apt-get update
+    $ sudo apt-get install build-essential golang-1.7 devscripts debhelper
+
+Create the source packages:
+
+    $ go run build/ci.go debsrc -workdir dist
+
+Then go into the source package directory for your running distribution and build the package:
+
+    $ cd dist/ethereum-unstable-1.5.0+xenial
+    $ dpkg-buildpackage
+
+Built packages are placed in the dist/ directory.
+
+    $ cd ..
+    $ dpkg-deb -c geth-unstable_1.5.0+xenial_amd64.deb

build/ci.go

@@ -120,8 +120,6 @@ func main() {
 		doArchive(os.Args[2:])
 	case "debsrc":
 		doDebianSource(os.Args[2:])
-	case "travis-debsrc":
-		doTravisDebianSource(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
 	default:
@@ -132,8 +130,8 @@ func main() {
 // Compiling
 
 func doInstall(cmdline []string) {
-	commitHash := flag.String("gitcommit", "", "Git commit hash embedded into binary.")
 	flag.CommandLine.Parse(cmdline)
+	env := build.Env()
 
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
@@ -150,13 +148,17 @@ func doInstall(cmdline []string) {
 		packages = flag.Args()
 	}
 
-	goinstall := goTool("install", makeBuildFlags(*commitHash)...)
+	goinstall := goTool("install", buildFlags(env)...)
 	goinstall.Args = append(goinstall.Args, "-v")
 	goinstall.Args = append(goinstall.Args, packages...)
 	build.MustRun(goinstall)
 }
 
-func makeBuildFlags(commitHash string) (flags []string) {
+func buildFlags(env build.Environment) (flags []string) {
+	if os.Getenv("GO_OPENCL") != "" {
+		flags = append(flags, "-tags", "opencl")
+	}
+
 	// Since Go 1.5, the separator char for link time assignments
 	// is '=' and using ' ' prints a warning. However, Go < 1.5 does
 	// not support using '='.
@@ -164,26 +166,9 @@ func makeBuildFlags(commitHash string) (flags []string) {
 	if runtime.Version() > "go1.5" || strings.Contains(runtime.Version(), "devel") {
 		sep = "="
 	}
-
+	// Set gitCommit constant via link-time assignment.
-	if os.Getenv("GO_OPENCL") != "" {
+	if env.Commit != "" {
-		flags = append(flags, "-tags", "opencl")
+		flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+env.Commit)
-	}
-
-	// Set gitCommit constant via link-time assignment. If this is a git checkout, we can
-	// just get the current commit hash through git. Otherwise we fall back to the hash
-	// that was passed as -gitcommit.
-	//
-	// -gitcommit is required for Debian package builds. The source package doesn't
-	// contain .git but we still want to embed the commit hash into the packaged binary.
-	// The hash is rendered into the debian/rules build script when the source package is
-	// created.
-	if _, err := os.Stat(filepath.Join(".git", "HEAD")); !os.IsNotExist(err) {
-		if c := build.GitCommit(); c != "" {
-			commitHash = c
-		}
-	}
-	if commitHash != "" {
-		flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+commitHash)
 	}
 	return flags
 }
@@ -227,6 +212,9 @@ func doTest(cmdline []string) {
 
 	// Run the actual tests.
 	gotest := goTool("test")
+	// Test a single package at a time. CI builders are slow
+	// and some tests run into timeouts under load.
+	gotest.Args = append(gotest.Args, "-p", "1")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -250,7 +238,11 @@ func doArchive(cmdline []string) {
 	default:
 		log.Fatal("unknown archive type: ", atype)
 	}
-	base := makeArchiveBasename()
+
+	env := build.Env()
+	maybeSkipArchive(env)
+
+	base := archiveBasename(env)
 	if err := build.WriteArchive("geth-"+base, ext, gethArchiveFiles); err != nil {
 		log.Fatal(err)
 	}
@@ -259,36 +251,41 @@ func doArchive(cmdline []string) {
 	}
 }
 
-func makeArchiveBasename() string {
+func archiveBasename(env build.Environment) string {
 	// date := time.Now().UTC().Format("200601021504")
 	platform := runtime.GOOS + "-" + runtime.GOARCH
 	archive := platform + "-" + build.VERSION()
-	if commit := build.GitCommit(); commit != "" {
+	if env.Commit != "" {
-		archive += "-" + commit[:8]
+		archive += "-" + env.Commit[:8]
 	}
 	return archive
 }
 
+// skips archiving for some build configurations.
+func maybeSkipArchive(env build.Environment) {
+	if env.IsPullRequest {
+		log.Printf("skipping because this is a PR build")
+		os.Exit(0)
+	}
+	if env.Branch != "develop" && !strings.HasPrefix(env.Tag, "v1.") {
+		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
+		os.Exit(0)
+	}
+}
+
 // Debian Packaging
 
-// CLI entry point for Travis CI.
+func doDebianSource(cmdline []string) {
-func doTravisDebianSource(cmdline []string) {
+	var (
+		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
+		upload  = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+		workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
+		now     = time.Now()
+	)
 	flag.CommandLine.Parse(cmdline)
-
+	*workdir = makeWorkdir(*workdir)
-	// Package only whitelisted branches.
+	env := build.Env()
-	switch {
+	maybeSkipArchive(env)
-	case os.Getenv("TRAVIS_REPO_SLUG") != "ethereum/go-ethereum":
-		log.Printf("skipping because this is a fork build")
-		return
-	case os.Getenv("TRAVIS_PULL_REQUEST") != "false":
-		log.Printf("skipping because this is a PR build")
-		return
-	case os.Getenv("TRAVIS_BRANCH") != "develop" && !strings.HasPrefix(os.Getenv("TRAVIS_TAG"), "v1."):
-		log.Printf("skipping because branch %q tag %q is not on the whitelist",
-			os.Getenv("TRAVIS_BRANCH"),
-			os.Getenv("TRAVIS_TAG"))
-		return
-	}
 
 	// Import the signing key.
 	if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
@@ -301,46 +298,16 @@ func doTravisDebianSource(cmdline []string) {
 		build.MustRun(gpg)
 	}
 
-	// Assign unstable status to non-tag builds.
+	// Create the packages.
-	unstable := "true"
-	if os.Getenv("TRAVIS_BRANCH") != "develop" && os.Getenv("TRAVIS_TAG") != "" {
-		unstable = "false"
-	}
-
-	doDebianSource([]string{
-		"-signer", "Felix Lange (Geth CI Testing Key) <fjl@twurst.com>",
-		"-buildnum", os.Getenv("TRAVIS_BUILD_NUMBER"),
-		"-upload", "ppa:lp-fjl/geth-ci-testing",
-		"-unstable", unstable,
-	})
-}
-
-// CLI entry point for doing packaging locally.
-func doDebianSource(cmdline []string) {
-	var (
-		signer   = flag.String("signer", "", `Signing key name, also used as package author`)
-		upload   = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
-		buildnum = flag.String("buildnum", "", `Build number (included in version)`)
-		unstable = flag.Bool("unstable", false, `Use package name suffix "-unstable"`)
-		now      = time.Now()
-	)
-	flag.CommandLine.Parse(cmdline)
-
-	// Create the debian worktree in /tmp.
-	tmpdir, err := ioutil.TempDir("", "eth-deb-build-")
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	for _, distro := range debDistros {
-		meta := newDebMetadata(distro, *signer, *buildnum, *unstable, now)
+		meta := newDebMetadata(distro, *signer, env, now)
-		pkgdir := stageDebianSource(tmpdir, meta)
+		pkgdir := stageDebianSource(*workdir, meta)
 		debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
 		debuild.Dir = pkgdir
 		build.MustRun(debuild)
 
 		changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
-		changes = filepath.Join(tmpdir, changes)
+		changes = filepath.Join(*workdir, changes)
 		if *signer != "" {
 			build.MustRunCommand("debsign", changes)
 		}
@@ -350,35 +317,53 @@ func doDebianSource(cmdline []string) {
 	}
 }
 
-type debExecutable struct {
+func makeWorkdir(wdflag string) string {
-	Name, Description string
+	var err error
+	if wdflag != "" {
+		err = os.MkdirAll(wdflag, 0744)
+	} else {
+		wdflag, err = ioutil.TempDir("", "eth-deb-build-")
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+	return wdflag
+}
+
+func isUnstableBuild(env build.Environment) bool {
+	if env.Branch != "develop" && env.Tag != "" {
+		return false
+	}
+	return true
 }
 
 type debMetadata struct {
+	Env build.Environment
+
 	// go-ethereum version being built. Note that this
 	// is not the debian package version. The package version
 	// is constructed by VersionString.
 	Version string
 
-	Author               string // "name <email>", also selects signing key
+	Author       string // "name <email>", also selects signing key
-	Buildnum             string // build number
+	Distro, Time string
-	Distro, Commit, Time string
+	Executables  []debExecutable
-	Executables          []debExecutable
-	Unstable             bool
 }
 
-func newDebMetadata(distro, author, buildnum string, unstable bool, t time.Time) debMetadata {
+type debExecutable struct {
+	Name, Description string
+}
+
+func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata {
 	if author == "" {
 		// No signing key, use default author.
 		author = "Ethereum Builds <fjl@ethereum.org>"
 	}
 	return debMetadata{
-		Unstable:    unstable,
+		Env:         env,
 		Author:      author,
 		Distro:      distro,
-		Commit:      build.GitCommit(),
 		Version:     build.VERSION(),
-		Buildnum:    buildnum,
 		Time:        t.Format(time.RFC1123Z),
 		Executables: debExecutables,
 	}
@@ -387,7 +372,7 @@ func newDebMetadata(distro, author, buildnum string, unstable bool, t time.Time)
 // Name returns the name of the metapackage that depends
 // on all executable packages.
 func (meta debMetadata) Name() string {
-	if meta.Unstable {
+	if isUnstableBuild(meta.Env) {
 		return "ethereum-unstable"
 	}
 	return "ethereum"
@@ -396,8 +381,8 @@ func (meta debMetadata) Name() string {
 // VersionString returns the debian version of the packages.
 func (meta debMetadata) VersionString() string {
 	vsn := meta.Version
-	if meta.Buildnum != "" {
+	if meta.Env.Buildnum != "" {
-		vsn += "+build" + meta.Buildnum
+		vsn += "+build" + meta.Env.Buildnum
 	}
 	if meta.Distro != "" {
 		vsn += "+" + meta.Distro
@@ -416,7 +401,7 @@ func (meta debMetadata) ExeList() string {
 
 // ExeName returns the package name of an executable package.
 func (meta debMetadata) ExeName(exe debExecutable) string {
-	if meta.Unstable {
+	if isUnstableBuild(meta.Env) {
 		return exe.Name + "-unstable"
 	}
 	return exe.Name
@@ -425,7 +410,7 @@ func (meta debMetadata) ExeName(exe debExecutable) string {
 // ExeConflicts returns the content of the Conflicts field
 // for executable packages.
 func (meta debMetadata) ExeConflicts(exe debExecutable) string {
-	if meta.Unstable {
+	if isUnstableBuild(meta.Env) {
 		// Set up the conflicts list so that the *-unstable packages
 		// cannot be installed alongside the regular version.
 		//
@@ -458,8 +443,8 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
 	build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
 	build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
 	for _, exe := range meta.Executables {
-		install := filepath.Join(debian, exe.Name+".install")
+		install := filepath.Join(debian, meta.ExeName(exe)+".install")
-		docs := filepath.Join(debian, exe.Name+".docs")
+		docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
 		build.Render("build/deb.install", install, 0644, exe)
 		build.Render("build/deb.docs", docs, 0644, exe)
 	}
@@ -470,18 +455,19 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
 // Cross compilation
 
 func doXgo(cmdline []string) {
+	flag.CommandLine.Parse(cmdline)
+	env := build.Env()
+
 	// Make sure xgo is available for cross compilation
 	gogetxgo := goTool("get", "github.com/karalabe/xgo")
 	build.MustRun(gogetxgo)
 
 	// Execute the actual cross compilation
-	pkg := cmdline[len(cmdline)-1]
+	xgo := xgoTool(append(buildFlags(env), flag.Args()...))
-	args := append(cmdline[:len(cmdline)-1], makeBuildFlags("")...)
+	build.MustRun(xgo)
-
-	build.MustRun(xgoTool(append(args, pkg)...))
 }
 
-func xgoTool(args ...string) *exec.Cmd {
+func xgoTool(args []string) *exec.Cmd {
 	cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...)
 	cmd.Env = []string{
 		"GOPATH=" + build.GOPATH(),

build/deb.changelog

@@ -1,5 +1,5 @@
 {{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
 
-  * git build of {{.Commit}}
+  * git build of {{.Env.Commit}}
 
  -- {{.Author}}  {{.Time}}

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.6
+Build-Depends: debhelper (>= 8.0.0), golang-1.7
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.6/bin/go run build/ci.go install -gitcommit {{.Commit}}
+	build/env.sh /usr/lib/go-1.7/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/update-license.go

@@ -49,7 +49,6 @@ var (
 		// don't relicense vendored sources
 		"crypto/sha3/", "crypto/ecies/", "logger/glog/",
 		"crypto/secp256k1/curve.go",
-		"trie/arc.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.

cmd/evm/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"gopkg.in/urfave/cli.v1"
@@ -141,7 +142,9 @@ func run(ctx *cli.Context) error {
 		)
 	} else {
 		receiver := statedb.CreateAccount(common.StringToAddress("receiver"))
-		receiver.SetCode(common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name)))
+
+		code := common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))
+		receiver.SetCode(crypto.Keccak256Hash(code), code)
 		ret, err = vmenv.Call(
 			sender,
 			receiver.Address(),

cmd/geth/main.go

@@ -50,7 +50,7 @@ const (
 	clientIdentifier = "Geth"   // Client identifier to advertise over the network
 	versionMajor     = 1        // Major version component of the current release
 	versionMinor     = 4        // Minor version component of the current release
-	versionPatch     = 12       // Patch version component of the current release
+	versionPatch     = 15       // Patch version component of the current release
 	versionMeta      = "stable" // Version metadata to append to the version string
 
 	versionOracle = "0xfa7b9770ca4cb04296cac84f37736d4041251cdf" // Ethereum address of the Geth release oracle

cmd/utils/cmd.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"os/signal"
 	"regexp"
+	"runtime"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -52,10 +53,16 @@ func openLogFile(Datadir string, filename string) *os.File {
 // is redirected to a different file.
 func Fatalf(format string, args ...interface{}) {
 	w := io.MultiWriter(os.Stdout, os.Stderr)
-	outf, _ := os.Stdout.Stat()
+	if runtime.GOOS == "windows" {
-	errf, _ := os.Stderr.Stat()
+		// The SameFile check below doesn't work on Windows.
-	if outf != nil && errf != nil && os.SameFile(outf, errf) {
+		// stdout is unlikely to get redirected though, so just print there.
-		w = os.Stderr
+		w = os.Stdout
+	} else {
+		outf, _ := os.Stdout.Stat()
+		errf, _ := os.Stderr.Stat()
+		if outf != nil && errf != nil && os.SameFile(outf, errf) {
+			w = os.Stderr
+		}
 	}
 	fmt.Fprintf(w, "Fatal: "+format+"\n", args...)
 	logger.Flush()

core/blockchain.go

@@ -95,10 +95,11 @@ type BlockChain struct {
 	currentBlock     *types.Block // Current head of the block chain
 	currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!)
 
-	bodyCache    *lru.Cache // Cache for the most recent block bodies
+	stateCache   *state.StateDB // State database to reuse between imports (contains state cache)
-	bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format
+	bodyCache    *lru.Cache     // Cache for the most recent block bodies
-	blockCache   *lru.Cache // Cache for the most recent entire blocks
+	bodyRLPCache *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
-	futureBlocks *lru.Cache // future blocks are blocks added for later processing
+	blockCache   *lru.Cache     // Cache for the most recent entire blocks
+	futureBlocks *lru.Cache     // future blocks are blocks added for later processing
 
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
@@ -198,11 +199,18 @@ func (self *BlockChain) loadLastState() error {
 			self.currentFastBlock = block
 		}
 	}
+	// Initialize a statedb cache to ensure singleton account bloom filter generation
+	statedb, err := state.New(self.currentBlock.Root(), self.chainDb)
+	if err != nil {
+		return err
+	}
+	self.stateCache = statedb
+	self.stateCache.GetAccount(common.Address{})
+
 	// Issue a status log and return
 	headerTd := self.GetTd(self.hc.CurrentHeader().Hash())
 	blockTd := self.GetTd(self.currentBlock.Hash())
 	fastTd := self.GetTd(self.currentFastBlock.Hash())
-
 	glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", self.hc.CurrentHeader().Number, self.hc.CurrentHeader().Hash().Bytes()[:4], headerTd)
 	glog.V(logger.Info).Infof("Last block: #%d [%x…] TD=%v", self.currentBlock.Number(), self.currentBlock.Hash().Bytes()[:4], blockTd)
 	glog.V(logger.Info).Infof("Fast block: #%d [%x…] TD=%v", self.currentFastBlock.Number(), self.currentFastBlock.Hash().Bytes()[:4], fastTd)
@@ -349,7 +357,12 @@ func (self *BlockChain) AuxValidator() pow.PoW { return self.pow }
 
 // State returns a new mutable state based on the current HEAD block.
 func (self *BlockChain) State() (*state.StateDB, error) {
-	return state.New(self.CurrentBlock().Root(), self.chainDb)
+	return self.StateAt(self.CurrentBlock().Root())
+}
+
+// StateAt returns a new mutable state based on a particular point in time.
+func (self *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
+	return self.stateCache.New(root)
 }
 
 // Reset purges the entire blockchain, restoring it to its genesis state.
@@ -817,7 +830,6 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 		tstart        = time.Now()
 
 		nonceChecked = make([]bool, len(chain))
-		statedb      *state.StateDB
 	)
 
 	// Start the parallel nonce verifier.
@@ -884,29 +896,30 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 
 		// Create a new statedb using the parent block and report an
 		// error if it fails.
-		if statedb == nil {
+		switch {
-			statedb, err = state.New(self.GetBlock(block.ParentHash()).Root(), self.chainDb)
+		case i == 0:
-		} else {
+			err = self.stateCache.Reset(self.GetBlock(block.ParentHash()).Root())
-			err = statedb.Reset(chain[i-1].Root())
+		default:
+			err = self.stateCache.Reset(chain[i-1].Root())
 		}
 		if err != nil {
 			reportBlock(block, err)
 			return i, err
 		}
 		// Process block using the parent state as reference point.
-		receipts, logs, usedGas, err := self.processor.Process(block, statedb, self.config.VmConfig)
+		receipts, logs, usedGas, err := self.processor.Process(block, self.stateCache, self.config.VmConfig)
 		if err != nil {
 			reportBlock(block, err)
 			return i, err
 		}
 		// Validate the state using the default validator
-		err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash()), statedb, receipts, usedGas)
+		err = self.Validator().ValidateState(block, self.GetBlock(block.ParentHash()), self.stateCache, receipts, usedGas)
 		if err != nil {
 			reportBlock(block, err)
 			return i, err
 		}
 		// Write state changes to database
-		_, err = statedb.Commit()
+		_, err = self.stateCache.Commit()
 		if err != nil {
 			return i, err
 		}

core/chain_makers_test.go

@@ -79,7 +79,7 @@ func ExampleGenerateChain() {
 	evmux := &event.TypeMux{}
 	blockchain, _ := NewBlockChain(db, MakeChainConfig(), FakePow{}, evmux)
 	if i, err := blockchain.InsertChain(chain); err != nil {
-		fmt.Printf("insert error (block %d): %v\n", i, err)
+		fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err)
 		return
 	}
 

core/execution.go

@@ -27,14 +27,14 @@ import (
 
 // Call executes within the given contract
 func Call(env vm.Environment, caller vm.ContractRef, addr common.Address, input []byte, gas, gasPrice, value *big.Int) (ret []byte, err error) {
-	ret, _, err = exec(env, caller, &addr, &addr, input, env.Db().GetCode(addr), gas, gasPrice, value)
+	ret, _, err = exec(env, caller, &addr, &addr, env.Db().GetCodeHash(addr), input, env.Db().GetCode(addr), gas, gasPrice, value)
 	return ret, err
 }
 
 // CallCode executes the given address' code as the given contract address
 func CallCode(env vm.Environment, caller vm.ContractRef, addr common.Address, input []byte, gas, gasPrice, value *big.Int) (ret []byte, err error) {
 	callerAddr := caller.Address()
-	ret, _, err = exec(env, caller, &callerAddr, &addr, input, env.Db().GetCode(addr), gas, gasPrice, value)
+	ret, _, err = exec(env, caller, &callerAddr, &addr, env.Db().GetCodeHash(addr), input, env.Db().GetCode(addr), gas, gasPrice, value)
 	return ret, err
 }
 
@@ -43,13 +43,13 @@ func DelegateCall(env vm.Environment, caller vm.ContractRef, addr common.Address
 	callerAddr := caller.Address()
 	originAddr := env.Origin()
 	callerValue := caller.Value()
-	ret, _, err = execDelegateCall(env, caller, &originAddr, &callerAddr, &addr, input, env.Db().GetCode(addr), gas, gasPrice, callerValue)
+	ret, _, err = execDelegateCall(env, caller, &originAddr, &callerAddr, &addr, env.Db().GetCodeHash(addr), input, env.Db().GetCode(addr), gas, gasPrice, callerValue)
 	return ret, err
 }
 
 // Create creates a new contract with the given code
 func Create(env vm.Environment, caller vm.ContractRef, code []byte, gas, gasPrice, value *big.Int) (ret []byte, address common.Address, err error) {
-	ret, address, err = exec(env, caller, nil, nil, nil, code, gas, gasPrice, value)
+	ret, address, err = exec(env, caller, nil, nil, crypto.Keccak256Hash(code), nil, code, gas, gasPrice, value)
 	// Here we get an error if we run into maximum stack depth,
 	// See: https://github.com/ethereum/yellowpaper/pull/131
 	// and YP definitions for CREATE instruction
@@ -59,7 +59,7 @@ func Create(env vm.Environment, caller vm.ContractRef, code []byte, gas, gasPric
 	return ret, address, err
 }
 
-func exec(env vm.Environment, caller vm.ContractRef, address, codeAddr *common.Address, input, code []byte, gas, gasPrice, value *big.Int) (ret []byte, addr common.Address, err error) {
+func exec(env vm.Environment, caller vm.ContractRef, address, codeAddr *common.Address, codeHash common.Hash, input, code []byte, gas, gasPrice, value *big.Int) (ret []byte, addr common.Address, err error) {
 	evm := env.Vm()
 	// Depth check execution. Fail if we're trying to execute above the
 	// limit.
@@ -105,7 +105,7 @@ func exec(env vm.Environment, caller vm.ContractRef, address, codeAddr *common.A
 	// EVM. The contract is a scoped environment for this execution context
 	// only.
 	contract := vm.NewContract(caller, to, value, gas, gasPrice)
-	contract.SetCallCode(codeAddr, code)
+	contract.SetCallCode(codeAddr, codeHash, code)
 	defer contract.Finalise()
 
 	ret, err = evm.Run(contract, input)
@@ -135,7 +135,7 @@ func exec(env vm.Environment, caller vm.ContractRef, address, codeAddr *common.A
 	return ret, addr, err
 }
 
-func execDelegateCall(env vm.Environment, caller vm.ContractRef, originAddr, toAddr, codeAddr *common.Address, input, code []byte, gas, gasPrice, value *big.Int) (ret []byte, addr common.Address, err error) {
+func execDelegateCall(env vm.Environment, caller vm.ContractRef, originAddr, toAddr, codeAddr *common.Address, codeHash common.Hash, input, code []byte, gas, gasPrice, value *big.Int) (ret []byte, addr common.Address, err error) {
 	evm := env.Vm()
 	// Depth check execution. Fail if we're trying to execute above the
 	// limit.
@@ -155,7 +155,7 @@ func execDelegateCall(env vm.Environment, caller vm.ContractRef, originAddr, toA
 
 	// Iinitialise a new contract and make initialise the delegate values
 	contract := vm.NewContract(caller, to, value, gas, gasPrice).AsDelegate()
-	contract.SetCallCode(codeAddr, code)
+	contract.SetCallCode(codeAddr, codeHash, code)
 	defer contract.Finalise()
 
 	ret, err = evm.Run(contract, input)

core/state/dump.go

@@ -21,9 +21,10 @@ import (
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
-type Account struct {
+type DumpAccount struct {
 	Balance  string            `json:"balance"`
 	Nonce    uint64            `json:"nonce"`
 	Root     string            `json:"root"`
@@ -32,40 +33,41 @@ type Account struct {
 	Storage  map[string]string `json:"storage"`
 }
 
-type World struct {
+type Dump struct {
-	Root     string             `json:"root"`
+	Root     string                 `json:"root"`
-	Accounts map[string]Account `json:"accounts"`
+	Accounts map[string]DumpAccount `json:"accounts"`
 }
 
-func (self *StateDB) RawDump() World {
+func (self *StateDB) RawDump() Dump {
-	world := World{
+	dump := Dump{
 		Root:     common.Bytes2Hex(self.trie.Root()),
-		Accounts: make(map[string]Account),
+		Accounts: make(map[string]DumpAccount),
 	}
 
 	it := self.trie.Iterator()
 	for it.Next() {
 		addr := self.trie.GetKey(it.Key)
-		stateObject, err := DecodeObject(common.BytesToAddress(addr), self.db, it.Value)
+		var data Account
-		if err != nil {
+		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
 			panic(err)
 		}
 
-		account := Account{
+		obj := NewObject(common.BytesToAddress(addr), data, nil)
-			Balance:  stateObject.balance.String(),
+		account := DumpAccount{
-			Nonce:    stateObject.nonce,
+			Balance:  data.Balance.String(),
-			Root:     common.Bytes2Hex(stateObject.Root()),
+			Nonce:    data.Nonce,
-			CodeHash: common.Bytes2Hex(stateObject.codeHash),
+			Root:     common.Bytes2Hex(data.Root[:]),
-			Code:     common.Bytes2Hex(stateObject.Code()),
+			CodeHash: common.Bytes2Hex(data.CodeHash),
+			Code:     common.Bytes2Hex(obj.Code(self.db)),
 			Storage:  make(map[string]string),
 		}
-		storageIt := stateObject.trie.Iterator()
+		storageIt := obj.getTrie(self.db).Iterator()
 		for storageIt.Next() {
 			account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
 		}
-		world.Accounts[common.Bytes2Hex(addr)] = account
+		dump.Accounts[common.Bytes2Hex(addr)] = account
 	}
-	return world
+	return dump
 }
 
 func (self *StateDB) Dump() []byte {
@@ -76,12 +78,3 @@ func (self *StateDB) Dump() []byte {
 
 	return json
 }
-
-// Debug stuff
-func (self *StateObject) CreateOutputForDiff() {
-	fmt.Printf("%x %x %x %x\n", self.Address(), self.Root(), self.balance.Bytes(), self.nonce)
-	it := self.trie.Iterator()
-	for it.Next() {
-		fmt.Printf("%x %x\n", it.Key, it.Value)
-	}
-}

core/state/iterator.go

@@ -76,7 +76,7 @@ func (it *NodeIterator) step() error {
 	}
 	// Initialize the iterator if we've just started
 	if it.stateIt == nil {
-		it.stateIt = trie.NewNodeIterator(it.state.trie.Trie)
+		it.stateIt = it.state.trie.NodeIterator()
 	}
 	// If we had data nodes previously, we surely have at least state nodes
 	if it.dataIt != nil {

core/state/managed_state.go

@@ -33,14 +33,14 @@ type ManagedState struct {
 
 	mu sync.RWMutex
 
-	accounts map[string]*account
+	accounts map[common.Address]*account
 }
 
 // ManagedState returns a new managed state with the statedb as it's backing layer
 func ManageState(statedb *StateDB) *ManagedState {
 	return &ManagedState{
 		StateDB:  statedb.Copy(),
-		accounts: make(map[string]*account),
+		accounts: make(map[common.Address]*account),
 	}
 }
 
@@ -103,7 +103,7 @@ func (ms *ManagedState) SetNonce(addr common.Address, nonce uint64) {
 	so := ms.GetOrNewStateObject(addr)
 	so.SetNonce(nonce)
 
-	ms.accounts[addr.Str()] = newAccount(so)
+	ms.accounts[addr] = newAccount(so)
 }
 
 // HasAccount returns whether the given address is managed or not
@@ -114,29 +114,28 @@ func (ms *ManagedState) HasAccount(addr common.Address) bool {
 }
 
 func (ms *ManagedState) hasAccount(addr common.Address) bool {
-	_, ok := ms.accounts[addr.Str()]
+	_, ok := ms.accounts[addr]
 	return ok
 }
 
 // populate the managed state
 func (ms *ManagedState) getAccount(addr common.Address) *account {
-	straddr := addr.Str()
+	if account, ok := ms.accounts[addr]; !ok {
-	if account, ok := ms.accounts[straddr]; !ok {
 		so := ms.GetOrNewStateObject(addr)
-		ms.accounts[straddr] = newAccount(so)
+		ms.accounts[addr] = newAccount(so)
 	} else {
 		// Always make sure the state account nonce isn't actually higher
 		// than the tracked one.
 		so := ms.StateDB.GetStateObject(addr)
-		if so != nil && uint64(len(account.nonces))+account.nstart < so.nonce {
+		if so != nil && uint64(len(account.nonces))+account.nstart < so.Nonce() {
-			ms.accounts[straddr] = newAccount(so)
+			ms.accounts[addr] = newAccount(so)
 		}
 
 	}
 
-	return ms.accounts[straddr]
+	return ms.accounts[addr]
 }
 
 func newAccount(so *StateObject) *account {
-	return &account{so, so.nonce, nil}
+	return &account{so, so.Nonce(), nil}
 }

core/state/managed_state_test.go

@@ -29,11 +29,12 @@ func create() (*ManagedState, *account) {
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := New(common.Hash{}, db)
 	ms := ManageState(statedb)
-	so := &StateObject{address: addr, nonce: 100}
+	so := &StateObject{address: addr}
-	ms.StateDB.stateObjects[addr.Str()] = so
+	so.SetNonce(100)
-	ms.accounts[addr.Str()] = newAccount(so)
+	ms.StateDB.stateObjects[addr] = so
+	ms.accounts[addr] = newAccount(so)
 
-	return ms, ms.accounts[addr.Str()]
+	return ms, ms.accounts[addr]
 }
 
 func TestNewNonce(t *testing.T) {
@@ -92,7 +93,7 @@ func TestRemoteNonceChange(t *testing.T) {
 	account.nonces = append(account.nonces, nn...)
 	nonce := ms.NewNonce(addr)
 
-	ms.StateDB.stateObjects[addr.Str()].nonce = 200
+	ms.StateDB.stateObjects[addr].data.Nonce = 200
 	nonce = ms.NewNonce(addr)
 	if nonce != 200 {
 		t.Error("expected nonce after remote update to be", 201, "got", nonce)
@@ -100,7 +101,7 @@ func TestRemoteNonceChange(t *testing.T) {
 	ms.NewNonce(addr)
 	ms.NewNonce(addr)
 	ms.NewNonce(addr)
-	ms.StateDB.stateObjects[addr.Str()].nonce = 200
+	ms.StateDB.stateObjects[addr].data.Nonce = 200
 	nonce = ms.NewNonce(addr)
 	if nonce != 204 {
 		t.Error("expected nonce after remote update to be", 201, "got", nonce)

core/state/state_object.go

@@ -57,143 +57,200 @@ func (self Storage) Copy() Storage {
 	return cpy
 }
 
+// StateObject represents an Ethereum account which is being modified.
+//
+// The usage pattern is as follows:
+// First you need to obtain a state object.
+// Account values can be accessed and modified through the object.
+// Finally, call CommitTrie to write the modified storage trie into a database.
 type StateObject struct {
-	db   trie.Database // State database for storing state changes
+	address common.Address // Ethereum address of this account
-	trie *trie.SecureTrie
+	data    Account
 
-	// Address belonging to this account
+	// DB error.
-	address common.Address
+	// State objects are used by the consensus core and VM which are
-	// The balance of the account
+	// unable to deal with database-level errors. Any error that occurs
-	balance *big.Int
+	// during a database read is memoized here and will eventually be returned
-	// The nonce of the account
+	// by StateDB.Commit.
-	nonce uint64
+	dbErr error
-	// The code hash if code is present (i.e. a contract)
-	codeHash []byte
-	// The code for this account
-	code Code
-	// Temporarily initialisation code
-	initCode Code
-	// Cached storage (flushed when updated)
-	storage Storage
 
-	// Mark for deletion
+	// Write caches.
+	trie *trie.SecureTrie // storage trie, which becomes non-nil on first access
+	code Code             // contract bytecode, which gets set when code is loaded
+
+	cachedStorage Storage // Storage entry cache to avoid duplicate reads
+	dirtyStorage  Storage // Storage entries that need to be flushed to disk
+
+	// Cache flags.
 	// When an object is marked for deletion it will be delete from the trie
 	// during the "update" phase of the state transition
-	remove  bool
+	dirtyCode bool // true if the code was updated
-	deleted bool
+	remove    bool
-	dirty   bool
+	deleted   bool
+	onDirty   func(addr common.Address) // Callback method to mark a state object newly dirty
 }
 
-func NewStateObject(address common.Address, db trie.Database) *StateObject {
+// Account is the Ethereum consensus representation of accounts.
-	object := &StateObject{
+// These objects are stored in the main account trie.
-		db:       db,
+type Account struct {
-		address:  address,
+	Nonce    uint64
-		balance:  new(big.Int),
+	Balance  *big.Int
-		dirty:    true,
+	Root     common.Hash // merkle root of the storage trie
-		codeHash: emptyCodeHash,
+	CodeHash []byte
-		storage:  make(Storage),
+}
+
+// NewObject creates a state object.
+func NewObject(address common.Address, data Account, onDirty func(addr common.Address)) *StateObject {
+	if data.Balance == nil {
+		data.Balance = new(big.Int)
+	}
+	if data.CodeHash == nil {
+		data.CodeHash = emptyCodeHash
+	}
+	return &StateObject{address: address, data: data, cachedStorage: make(Storage), dirtyStorage: make(Storage), onDirty: onDirty}
+}
+
+// EncodeRLP implements rlp.Encoder.
+func (c *StateObject) EncodeRLP(w io.Writer) error {
+	return rlp.Encode(w, c.data)
+}
+
+// setError remembers the first non-nil error it is called with.
+func (self *StateObject) setError(err error) {
+	if self.dbErr == nil {
+		self.dbErr = err
 	}
-	object.trie, _ = trie.NewSecure(common.Hash{}, db)
-	return object
 }
 
 func (self *StateObject) MarkForDeletion() {
 	self.remove = true
-	self.dirty = true
+	if self.onDirty != nil {
-
+		self.onDirty(self.Address())
+		self.onDirty = nil
+	}
 	if glog.V(logger.Core) {
-		glog.Infof("%x: #%d %v X\n", self.Address(), self.nonce, self.balance)
+		glog.Infof("%x: #%d %v X\n", self.Address(), self.Nonce(), self.Balance())
 	}
 }
 
-func (c *StateObject) getAddr(addr common.Hash) common.Hash {
+func (c *StateObject) getTrie(db trie.Database) *trie.SecureTrie {
-	var ret []byte
+	if c.trie == nil {
-	rlp.DecodeBytes(c.trie.Get(addr[:]), &ret)
+		var err error
-	return common.BytesToHash(ret)
+		c.trie, err = trie.NewSecure(c.data.Root, db)
-}
+		if err != nil {
-
+			c.trie, _ = trie.NewSecure(common.Hash{}, db)
-func (c *StateObject) setAddr(addr, value common.Hash) {
+			c.setError(fmt.Errorf("can't create storage trie: %v", err))
-	v, err := rlp.EncodeToBytes(bytes.TrimLeft(value[:], "\x00"))
-	if err != nil {
-		// if RLPing failed we better panic and not fail silently. This would be considered a consensus issue
-		panic(err)
-	}
-	c.trie.Update(addr[:], v)
-}
-
-func (self *StateObject) Storage() Storage {
-	return self.storage
-}
-
-func (self *StateObject) GetState(key common.Hash) common.Hash {
-	value, exists := self.storage[key]
-	if !exists {
-		value = self.getAddr(key)
-		if (value != common.Hash{}) {
-			self.storage[key] = value
 		}
 	}
+	return c.trie
+}
 
+// GetState returns a value in account storage.
+func (self *StateObject) GetState(db trie.Database, key common.Hash) common.Hash {
+	value, exists := self.cachedStorage[key]
+	if exists {
+		return value
+	}
+	// Load from DB in case it is missing.
+	tr := self.getTrie(db)
+	var ret []byte
+	rlp.DecodeBytes(tr.Get(key[:]), &ret)
+	value = common.BytesToHash(ret)
+	if (value != common.Hash{}) {
+		self.cachedStorage[key] = value
+	}
 	return value
 }
 
+// SetState updates a value in account storage.
 func (self *StateObject) SetState(key, value common.Hash) {
-	self.storage[key] = value
+	self.cachedStorage[key] = value
-	self.dirty = true
+	self.dirtyStorage[key] = value
-}
 
-// Update updates the current cached storage to the trie
+	if self.onDirty != nil {
-func (self *StateObject) Update() {
+		self.onDirty(self.Address())
-	for key, value := range self.storage {
+		self.onDirty = nil
-		if (value == common.Hash{}) {
-			self.trie.Delete(key[:])
-			continue
-		}
-		self.setAddr(key, value)
 	}
 }
 
+// updateTrie writes cached storage modifications into the object's storage trie.
+func (self *StateObject) updateTrie(db trie.Database) {
+	tr := self.getTrie(db)
+	for key, value := range self.dirtyStorage {
+		delete(self.dirtyStorage, key)
+		if (value == common.Hash{}) {
+			tr.Delete(key[:])
+			continue
+		}
+		// Encoding []byte cannot fail, ok to ignore the error.
+		v, _ := rlp.EncodeToBytes(bytes.TrimLeft(value[:], "\x00"))
+		tr.Update(key[:], v)
+	}
+}
+
+// UpdateRoot sets the trie root to the current root hash of
+func (self *StateObject) UpdateRoot(db trie.Database) {
+	self.updateTrie(db)
+	self.data.Root = self.trie.Hash()
+}
+
+// CommitTrie the storage trie of the object to dwb.
+// This updates the trie root.
+func (self *StateObject) CommitTrie(db trie.Database, dbw trie.DatabaseWriter) error {
+	self.updateTrie(db)
+	if self.dbErr != nil {
+		fmt.Println("dbErr:", self.dbErr)
+		return self.dbErr
+	}
+	root, err := self.trie.CommitTo(dbw)
+	if err == nil {
+		self.data.Root = root
+	}
+	return err
+}
+
 func (c *StateObject) AddBalance(amount *big.Int) {
-	c.SetBalance(new(big.Int).Add(c.balance, amount))
+	if amount.Cmp(common.Big0) == 0 {
+		return
+	}
+	c.SetBalance(new(big.Int).Add(c.Balance(), amount))
 
 	if glog.V(logger.Core) {
-		glog.Infof("%x: #%d %v (+ %v)\n", c.Address(), c.nonce, c.balance, amount)
+		glog.Infof("%x: #%d %v (+ %v)\n", c.Address(), c.Nonce(), c.Balance(), amount)
 	}
 }
 
 func (c *StateObject) SubBalance(amount *big.Int) {
-	c.SetBalance(new(big.Int).Sub(c.balance, amount))
+	if amount.Cmp(common.Big0) == 0 {
+		return
+	}
+	c.SetBalance(new(big.Int).Sub(c.Balance(), amount))
 
 	if glog.V(logger.Core) {
-		glog.Infof("%x: #%d %v (- %v)\n", c.Address(), c.nonce, c.balance, amount)
+		glog.Infof("%x: #%d %v (- %v)\n", c.Address(), c.Nonce(), c.Balance(), amount)
 	}
 }
 
-func (c *StateObject) SetBalance(amount *big.Int) {
+func (self *StateObject) SetBalance(amount *big.Int) {
-	c.balance = amount
+	self.data.Balance = amount
-	c.dirty = true
+	if self.onDirty != nil {
-}
+		self.onDirty(self.Address())
-
+		self.onDirty = nil
-func (c *StateObject) St() Storage {
+	}
-	return c.storage
 }
 
 // Return the gas back to the origin. Used by the Virtual machine or Closures
 func (c *StateObject) ReturnGas(gas, price *big.Int) {}
 
-func (self *StateObject) Copy() *StateObject {
+func (self *StateObject) Copy(db trie.Database, onDirty func(addr common.Address)) *StateObject {
-	stateObject := NewStateObject(self.Address(), self.db)
+	stateObject := NewObject(self.address, self.data, onDirty)
-	stateObject.balance.Set(self.balance)
-	stateObject.codeHash = common.CopyBytes(self.codeHash)
-	stateObject.nonce = self.nonce
 	stateObject.trie = self.trie
 	stateObject.code = self.code
-	stateObject.initCode = common.CopyBytes(self.initCode)
+	stateObject.dirtyStorage = self.dirtyStorage.Copy()
-	stateObject.storage = self.storage.Copy()
+	stateObject.cachedStorage = self.dirtyStorage.Copy()
 	stateObject.remove = self.remove
-	stateObject.dirty = self.dirty
+	stateObject.dirtyCode = self.dirtyCode
 	stateObject.deleted = self.deleted
-
 	return stateObject
 }
 
@@ -201,40 +258,55 @@ func (self *StateObject) Copy() *StateObject {
 // Attribute accessors
 //
 
-func (self *StateObject) Balance() *big.Int {
-	return self.balance
-}
-
 // Returns the address of the contract/account
 func (c *StateObject) Address() common.Address {
 	return c.address
 }
 
-func (self *StateObject) Trie() *trie.SecureTrie {
+// Code returns the contract code associated with this object, if any.
-	return self.trie
+func (self *StateObject) Code(db trie.Database) []byte {
-}
+	if self.code != nil {
-
+		return self.code
-func (self *StateObject) Root() []byte {
+	}
-	return self.trie.Root()
+	if bytes.Equal(self.CodeHash(), emptyCodeHash) {
-}
+		return nil
-
+	}
-func (self *StateObject) Code() []byte {
+	code, err := db.Get(self.CodeHash())
-	return self.code
+	if err != nil {
-}
+		self.setError(fmt.Errorf("can't load code hash %x: %v", self.CodeHash(), err))
-
+	}
-func (self *StateObject) SetCode(code []byte) {
 	self.code = code
-	self.codeHash = crypto.Keccak256(code)
+	return code
-	self.dirty = true
+}
+
+func (self *StateObject) SetCode(codeHash common.Hash, code []byte) {
+	self.code = code
+	self.data.CodeHash = codeHash[:]
+	self.dirtyCode = true
+	if self.onDirty != nil {
+		self.onDirty(self.Address())
+		self.onDirty = nil
+	}
 }
 
 func (self *StateObject) SetNonce(nonce uint64) {
-	self.nonce = nonce
+	self.data.Nonce = nonce
-	self.dirty = true
+	if self.onDirty != nil {
+		self.onDirty(self.Address())
+		self.onDirty = nil
+	}
+}
+
+func (self *StateObject) CodeHash() []byte {
+	return self.data.CodeHash
+}
+
+func (self *StateObject) Balance() *big.Int {
+	return self.data.Balance
 }
 
 func (self *StateObject) Nonce() uint64 {
-	return self.nonce
+	return self.data.Nonce
 }
 
 // Never called, but must be present to allow StateObject to be used
@@ -246,7 +318,7 @@ func (self *StateObject) Value() *big.Int {
 
 func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) {
 	// When iterating over the storage check the cache first
-	for h, value := range self.storage {
+	for h, value := range self.cachedStorage {
 		cb(h, value)
 	}
 
@@ -254,44 +326,8 @@ func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) {
 	for it.Next() {
 		// ignore cached values
 		key := common.BytesToHash(self.trie.GetKey(it.Key))
-		if _, ok := self.storage[key]; !ok {
+		if _, ok := self.cachedStorage[key]; !ok {
 			cb(key, common.BytesToHash(it.Value))
 		}
 	}
 }
-
-type extStateObject struct {
-	Nonce    uint64
-	Balance  *big.Int
-	Root     common.Hash
-	CodeHash []byte
-}
-
-// EncodeRLP implements rlp.Encoder.
-func (c *StateObject) EncodeRLP(w io.Writer) error {
-	return rlp.Encode(w, []interface{}{c.nonce, c.balance, c.Root(), c.codeHash})
-}
-
-// DecodeObject decodes an RLP-encoded state object.
-func DecodeObject(address common.Address, db trie.Database, data []byte) (*StateObject, error) {
-	var (
-		obj = &StateObject{address: address, db: db, storage: make(Storage)}
-		ext extStateObject
-		err error
-	)
-	if err = rlp.DecodeBytes(data, &ext); err != nil {
-		return nil, err
-	}
-	if obj.trie, err = trie.NewSecure(ext.Root, db); err != nil {
-		return nil, err
-	}
-	if !bytes.Equal(ext.CodeHash, emptyCodeHash) {
-		if obj.code, err = db.Get(ext.CodeHash); err != nil {
-			return nil, fmt.Errorf("can't get code for hash %x: %v", ext.CodeHash, err)
-		}
-	}
-	obj.nonce = ext.Nonce
-	obj.balance = ext.Balance
-	obj.codeHash = ext.CodeHash
-	return obj, nil
-}

core/state/state_test.go

@@ -24,6 +24,7 @@ import (
 	checker "gopkg.in/check.v1"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
@@ -40,7 +41,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
 	obj1 := s.state.GetOrNewStateObject(toAddr([]byte{0x01}))
 	obj1.AddBalance(big.NewInt(22))
 	obj2 := s.state.GetOrNewStateObject(toAddr([]byte{0x01, 0x02}))
-	obj2.SetCode([]byte{3, 3, 3, 3, 3, 3, 3})
+	obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3})
 	obj3 := s.state.GetOrNewStateObject(toAddr([]byte{0x02}))
 	obj3.SetBalance(big.NewInt(44))
 
@@ -146,23 +147,23 @@ func TestSnapshot2(t *testing.T) {
 
 	// db, trie are already non-empty values
 	so0 := state.GetStateObject(stateobjaddr0)
-	so0.balance = big.NewInt(42)
+	so0.SetBalance(big.NewInt(42))
-	so0.nonce = 43
+	so0.SetNonce(43)
-	so0.SetCode([]byte{'c', 'a', 'f', 'e'})
+	so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'})
 	so0.remove = false
 	so0.deleted = false
-	so0.dirty = true
 	state.SetStateObject(so0)
-	state.Commit()
+
+	root, _ := state.Commit()
+	state.Reset(root)
 
 	// and one with deleted == true
 	so1 := state.GetStateObject(stateobjaddr1)
-	so1.balance = big.NewInt(52)
+	so1.SetBalance(big.NewInt(52))
-	so1.nonce = 53
+	so1.SetNonce(53)
-	so1.SetCode([]byte{'c', 'a', 'f', 'e', '2'})
+	so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'})
 	so1.remove = true
 	so1.deleted = true
-	so1.dirty = true
 	state.SetStateObject(so1)
 
 	so1 = state.GetStateObject(stateobjaddr1)
@@ -174,44 +175,50 @@ func TestSnapshot2(t *testing.T) {
 	state.Set(snapshot)
 
 	so0Restored := state.GetStateObject(stateobjaddr0)
-	so0Restored.GetState(storageaddr)
+	// Update lazily-loaded values before comparing.
-	so1Restored := state.GetStateObject(stateobjaddr1)
+	so0Restored.GetState(db, storageaddr)
+	so0Restored.Code(db)
 	// non-deleted is equal (restored)
 	compareStateObjects(so0Restored, so0, t)
+
 	// deleted should be nil, both before and after restore of state copy
+	so1Restored := state.GetStateObject(stateobjaddr1)
 	if so1Restored != nil {
-		t.Fatalf("deleted object not nil after restoring snapshot")
+		t.Fatalf("deleted object not nil after restoring snapshot: %+v", so1Restored)
 	}
 }
 
 func compareStateObjects(so0, so1 *StateObject, t *testing.T) {
-	if so0.address != so1.address {
+	if so0.Address() != so1.Address() {
 		t.Fatalf("Address mismatch: have %v, want %v", so0.address, so1.address)
 	}
-	if so0.balance.Cmp(so1.balance) != 0 {
+	if so0.Balance().Cmp(so1.Balance()) != 0 {
-		t.Fatalf("Balance mismatch: have %v, want %v", so0.balance, so1.balance)
+		t.Fatalf("Balance mismatch: have %v, want %v", so0.Balance(), so1.Balance())
 	}
-	if so0.nonce != so1.nonce {
+	if so0.Nonce() != so1.Nonce() {
-		t.Fatalf("Nonce mismatch: have %v, want %v", so0.nonce, so1.nonce)
+		t.Fatalf("Nonce mismatch: have %v, want %v", so0.Nonce(), so1.Nonce())
 	}
-	if !bytes.Equal(so0.codeHash, so1.codeHash) {
+	if so0.data.Root != so1.data.Root {
-		t.Fatalf("CodeHash mismatch: have %v, want %v", so0.codeHash, so1.codeHash)
+		t.Errorf("Root mismatch: have %x, want %x", so0.data.Root[:], so1.data.Root[:])
+	}
+	if !bytes.Equal(so0.CodeHash(), so1.CodeHash()) {
+		t.Fatalf("CodeHash mismatch: have %v, want %v", so0.CodeHash(), so1.CodeHash())
 	}
 	if !bytes.Equal(so0.code, so1.code) {
 		t.Fatalf("Code mismatch: have %v, want %v", so0.code, so1.code)
 	}
-	if !bytes.Equal(so0.initCode, so1.initCode) {
-		t.Fatalf("InitCode mismatch: have %v, want %v", so0.initCode, so1.initCode)
-	}
 
-	for k, v := range so1.storage {
+	if len(so1.cachedStorage) != len(so0.cachedStorage) {
-		if so0.storage[k] != v {
+		t.Errorf("Storage size mismatch: have %d, want %d", len(so1.cachedStorage), len(so0.cachedStorage))
-			t.Fatalf("Storage key %s mismatch: have %v, want %v", k, so0.storage[k], v)
+	}
+	for k, v := range so1.cachedStorage {
+		if so0.cachedStorage[k] != v {
+			t.Errorf("Storage key %x mismatch: have %v, want %v", k, so0.cachedStorage[k], v)
 		}
 	}
-	for k, v := range so0.storage {
+	for k, v := range so0.cachedStorage {
-		if so1.storage[k] != v {
+		if so1.cachedStorage[k] != v {
-			t.Fatalf("Storage key %s mismatch: have %v, want none.", k, v)
+			t.Errorf("Storage key %x mismatch: have %v, want none.", k, v)
 		}
 	}
 
@@ -221,7 +228,4 @@ func compareStateObjects(so0, so1 *StateObject, t *testing.T) {
 	if so0.deleted != so1.deleted {
 		t.Fatalf("Deleted mismatch: have %v, want %v", so0.deleted, so1.deleted)
 	}
-	if so0.dirty != so1.dirty {
-		t.Fatalf("Dirty mismatch: have %v, want %v", so0.dirty, so1.dirty)
-	}
 }

core/state/statedb.go

@@ -20,37 +20,56 @@ package state
 import (
 	"fmt"
 	"math/big"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 // The starting nonce determines the default nonce when new accounts are being
 // created.
 var StartingNonce uint64
 
+const (
+	// Number of past tries to keep. The arbitrarily chosen value here
+	// is max uncle depth + 1.
+	maxJournalLength = 8
+
+	// Number of codehash->size associations to keep.
+	codeSizeCacheSize = 100000
+)
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
 // * Contracts
 // * Accounts
 type StateDB struct {
-	db   ethdb.Database
+	db            ethdb.Database
-	trie *trie.SecureTrie
+	trie          *trie.SecureTrie
+	pastTries     []*trie.SecureTrie
+	codeSizeCache *lru.Cache
 
-	stateObjects map[string]*StateObject
+	// This map holds 'live' objects, which will get modified while processing a state transition.
+	stateObjects      map[common.Address]*StateObject
+	stateObjectsDirty map[common.Address]struct{}
 
+	// The refund counter, also used by state transitioning.
 	refund *big.Int
 
 	thash, bhash common.Hash
 	txIndex      int
 	logs         map[common.Hash]vm.Logs
 	logSize      uint
+
+	lock sync.Mutex
 }
 
 // Create a new state from a given trie
@@ -59,35 +78,84 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) {
 	if err != nil {
 		return nil, err
 	}
+	csc, _ := lru.New(codeSizeCacheSize)
 	return &StateDB{
-		db:           db,
+		db:                db,
-		trie:         tr,
+		trie:              tr,
-		stateObjects: make(map[string]*StateObject),
+		codeSizeCache:     csc,
-		refund:       new(big.Int),
+		stateObjects:      make(map[common.Address]*StateObject),
-		logs:         make(map[common.Hash]vm.Logs),
+		stateObjectsDirty: make(map[common.Address]struct{}),
+		refund:            new(big.Int),
+		logs:              make(map[common.Hash]vm.Logs),
+	}, nil
+}
+
+// New creates a new statedb by reusing any journalled tries to avoid costly
+// disk io.
+func (self *StateDB) New(root common.Hash) (*StateDB, error) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+
+	tr, err := self.openTrie(root)
+	if err != nil {
+		return nil, err
+	}
+	return &StateDB{
+		db:                self.db,
+		trie:              tr,
+		codeSizeCache:     self.codeSizeCache,
+		stateObjects:      make(map[common.Address]*StateObject),
+		stateObjectsDirty: make(map[common.Address]struct{}),
+		refund:            new(big.Int),
+		logs:              make(map[common.Hash]vm.Logs),
 	}, nil
 }
 
 // Reset clears out all emphemeral state objects from the state db, but keeps
 // the underlying state trie to avoid reloading data for the next operations.
 func (self *StateDB) Reset(root common.Hash) error {
-	var (
+	self.lock.Lock()
-		err error
+	defer self.lock.Unlock()
-		tr  = self.trie
+
-	)
+	tr, err := self.openTrie(root)
-	if self.trie.Hash() != root {
+	if err != nil {
-		if tr, err = trie.NewSecure(root, self.db); err != nil {
+		return err
-			return err
+	}
+	self.trie = tr
+	self.stateObjects = make(map[common.Address]*StateObject)
+	self.stateObjectsDirty = make(map[common.Address]struct{})
+	self.refund = new(big.Int)
+	self.thash = common.Hash{}
+	self.bhash = common.Hash{}
+	self.txIndex = 0
+	self.logs = make(map[common.Hash]vm.Logs)
+	self.logSize = 0
+
+	return nil
+}
+
+// openTrie creates a trie. It uses an existing trie if one is available
+// from the journal if available.
+func (self *StateDB) openTrie(root common.Hash) (*trie.SecureTrie, error) {
+	for i := len(self.pastTries) - 1; i >= 0; i-- {
+		if self.pastTries[i].Hash() == root {
+			tr := *self.pastTries[i]
+			return &tr, nil
 		}
 	}
-	*self = StateDB{
+	return trie.NewSecure(root, self.db)
-		db:           self.db,
+}
-		trie:         tr,
+
-		stateObjects: make(map[string]*StateObject),
+func (self *StateDB) pushTrie(t *trie.SecureTrie) {
-		refund:       new(big.Int),
+	self.lock.Lock()
-		logs:         make(map[common.Hash]vm.Logs),
+	defer self.lock.Unlock()
+
+	if len(self.pastTries) >= maxJournalLength {
+		copy(self.pastTries, self.pastTries[1:])
+		self.pastTries[len(self.pastTries)-1] = t
+	} else {
+		self.pastTries = append(self.pastTries, t)
 	}
-	return nil
 }
 
 func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) {
@@ -137,7 +205,7 @@ func (self *StateDB) GetAccount(addr common.Address) vm.Account {
 func (self *StateDB) GetBalance(addr common.Address) *big.Int {
 	stateObject := self.GetStateObject(addr)
 	if stateObject != nil {
-		return stateObject.balance
+		return stateObject.Balance()
 	}
 
 	return common.Big0
@@ -146,7 +214,7 @@ func (self *StateDB) GetBalance(addr common.Address) *big.Int {
 func (self *StateDB) GetNonce(addr common.Address) uint64 {
 	stateObject := self.GetStateObject(addr)
 	if stateObject != nil {
-		return stateObject.nonce
+		return stateObject.Nonce()
 	}
 
 	return StartingNonce
@@ -155,18 +223,43 @@ func (self *StateDB) GetNonce(addr common.Address) uint64 {
 func (self *StateDB) GetCode(addr common.Address) []byte {
 	stateObject := self.GetStateObject(addr)
 	if stateObject != nil {
-		return stateObject.code
+		code := stateObject.Code(self.db)
+		key := common.BytesToHash(stateObject.CodeHash())
+		self.codeSizeCache.Add(key, len(code))
+		return code
 	}
-
 	return nil
 }
 
+func (self *StateDB) GetCodeSize(addr common.Address) int {
+	stateObject := self.GetStateObject(addr)
+	if stateObject == nil {
+		return 0
+	}
+	key := common.BytesToHash(stateObject.CodeHash())
+	if cached, ok := self.codeSizeCache.Get(key); ok {
+		return cached.(int)
+	}
+	size := len(stateObject.Code(self.db))
+	if stateObject.dbErr == nil {
+		self.codeSizeCache.Add(key, size)
+	}
+	return size
+}
+
+func (self *StateDB) GetCodeHash(addr common.Address) common.Hash {
+	stateObject := self.GetStateObject(addr)
+	if stateObject == nil {
+		return common.Hash{}
+	}
+	return common.BytesToHash(stateObject.CodeHash())
+}
+
 func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
 	stateObject := self.GetStateObject(a)
 	if stateObject != nil {
-		return stateObject.GetState(b)
+		return stateObject.GetState(self.db, b)
 	}
-
 	return common.Hash{}
 }
 
@@ -199,7 +292,7 @@ func (self *StateDB) SetNonce(addr common.Address, nonce uint64) {
 func (self *StateDB) SetCode(addr common.Address, code []byte) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
-		stateObject.SetCode(code)
+		stateObject.SetCode(crypto.Keccak256Hash(code), code)
 	}
 }
 
@@ -214,8 +307,7 @@ func (self *StateDB) Delete(addr common.Address) bool {
 	stateObject := self.GetStateObject(addr)
 	if stateObject != nil {
 		stateObject.MarkForDeletion()
-		stateObject.balance = new(big.Int)
+		stateObject.data.Balance = new(big.Int)
-
 		return true
 	}
 
@@ -242,35 +334,36 @@ func (self *StateDB) DeleteStateObject(stateObject *StateObject) {
 
 	addr := stateObject.Address()
 	self.trie.Delete(addr[:])
-	//delete(self.stateObjects, addr.Str())
 }
 
-// Retrieve a state object given my the address. Nil if not found
+// Retrieve a state object given my the address. Returns nil if not found.
 func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObject) {
-	stateObject = self.stateObjects[addr.Str()]
+	// Prefer 'live' objects.
-	if stateObject != nil {
+	if obj := self.stateObjects[addr]; obj != nil {
-		if stateObject.deleted {
+		if obj.deleted {
-			stateObject = nil
+			return nil
 		}
-
+		return obj
-		return stateObject
 	}
 
-	data := self.trie.Get(addr[:])
+	// Load the object from the database.
-	if len(data) == 0 {
+	enc := self.trie.Get(addr[:])
+	if len(enc) == 0 {
 		return nil
 	}
-	stateObject, err := DecodeObject(addr, self.db, data)
+	var data Account
-	if err != nil {
+	if err := rlp.DecodeBytes(enc, &data); err != nil {
 		glog.Errorf("can't decode object at %x: %v", addr[:], err)
 		return nil
 	}
-	self.SetStateObject(stateObject)
+	// Insert into the live set.
-	return stateObject
+	obj := NewObject(addr, data, self.MarkStateObjectDirty)
+	self.SetStateObject(obj)
+	return obj
 }
 
 func (self *StateDB) SetStateObject(object *StateObject) {
-	self.stateObjects[object.Address().Str()] = object
+	self.stateObjects[object.Address()] = object
 }
 
 // Retrieve a state object or create a new state object if nil
@@ -288,15 +381,19 @@ func (self *StateDB) newStateObject(addr common.Address) *StateObject {
 	if glog.V(logger.Core) {
 		glog.Infof("(+) %x\n", addr)
 	}
-
+	obj := NewObject(addr, Account{}, self.MarkStateObjectDirty)
-	stateObject := NewStateObject(addr, self.db)
+	obj.SetNonce(StartingNonce) // sets the object to dirty
-	stateObject.SetNonce(StartingNonce)
+	self.stateObjects[addr] = obj
-	self.stateObjects[addr.Str()] = stateObject
+	return obj
-
-	return stateObject
 }
 
-// Creates creates a new state object and takes ownership. This is different from "NewStateObject"
+// MarkStateObjectDirty adds the specified object to the dirty map to avoid costly
+// state object cache iteration to find a handful of modified ones.
+func (self *StateDB) MarkStateObjectDirty(addr common.Address) {
+	self.stateObjectsDirty[addr] = struct{}{}
+}
+
+// Creates creates a new state object and takes ownership.
 func (self *StateDB) CreateStateObject(addr common.Address) *StateObject {
 	// Get previous (if any)
 	so := self.GetStateObject(addr)
@@ -305,7 +402,7 @@ func (self *StateDB) CreateStateObject(addr common.Address) *StateObject {
 
 	// If it existed set the balance to the new account
 	if so != nil {
-		newSo.balance = so.balance
+		newSo.data.Balance = so.data.Balance
 	}
 
 	return newSo
@@ -320,30 +417,43 @@ func (self *StateDB) CreateAccount(addr common.Address) vm.Account {
 //
 
 func (self *StateDB) Copy() *StateDB {
-	// ignore error - we assume state-to-be-copied always exists
+	self.lock.Lock()
-	state, _ := New(common.Hash{}, self.db)
+	defer self.lock.Unlock()
-	state.trie = self.trie
+
-	for k, stateObject := range self.stateObjects {
+	// Copy all the basic fields, initialize the memory ones
-		if stateObject.dirty {
+	state := &StateDB{
-			state.stateObjects[k] = stateObject.Copy()
+		db:                self.db,
-		}
+		trie:              self.trie,
+		pastTries:         self.pastTries,
+		codeSizeCache:     self.codeSizeCache,
+		stateObjects:      make(map[common.Address]*StateObject, len(self.stateObjectsDirty)),
+		stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)),
+		refund:            new(big.Int).Set(self.refund),
+		logs:              make(map[common.Hash]vm.Logs, len(self.logs)),
+		logSize:           self.logSize,
+	}
+	// Copy the dirty states and logs
+	for addr, _ := range self.stateObjectsDirty {
+		state.stateObjects[addr] = self.stateObjects[addr].Copy(self.db, state.MarkStateObjectDirty)
+		state.stateObjectsDirty[addr] = struct{}{}
 	}
-
-	state.refund.Set(self.refund)
-
 	for hash, logs := range self.logs {
 		state.logs[hash] = make(vm.Logs, len(logs))
 		copy(state.logs[hash], logs)
 	}
-	state.logSize = self.logSize
-
 	return state
 }
 
 func (self *StateDB) Set(state *StateDB) {
-	self.trie = state.trie
+	self.lock.Lock()
-	self.stateObjects = state.stateObjects
+	defer self.lock.Unlock()
 
+	self.db = state.db
+	self.trie = state.trie
+	self.pastTries = state.pastTries
+	self.stateObjects = state.stateObjects
+	self.stateObjectsDirty = state.stateObjectsDirty
+	self.codeSizeCache = state.codeSizeCache
 	self.refund = state.refund
 	self.logs = state.logs
 	self.logSize = state.logSize
@@ -358,14 +468,13 @@ func (self *StateDB) GetRefund() *big.Int {
 // goes into transaction receipts.
 func (s *StateDB) IntermediateRoot() common.Hash {
 	s.refund = new(big.Int)
-	for _, stateObject := range s.stateObjects {
+	for addr, _ := range s.stateObjectsDirty {
-		if stateObject.dirty {
+		stateObject := s.stateObjects[addr]
-			if stateObject.remove {
+		if stateObject.remove {
-				s.DeleteStateObject(stateObject)
+			s.DeleteStateObject(stateObject)
-			} else {
+		} else {
-				stateObject.Update()
+			stateObject.UpdateRoot(s.db)
-				s.UpdateStateObject(stateObject)
+			s.UpdateStateObject(stateObject)
-			}
 		}
 	}
 	return s.trie.Hash()
@@ -380,15 +489,15 @@ func (s *StateDB) DeleteSuicides() {
 	// Reset refund so that any used-gas calculations can use
 	// this method.
 	s.refund = new(big.Int)
-	for _, stateObject := range s.stateObjects {
+	for addr, _ := range s.stateObjectsDirty {
-		if stateObject.dirty {
+		stateObject := s.stateObjects[addr]
-			// If the object has been removed by a suicide
+
-			// flag the object as deleted.
+		// If the object has been removed by a suicide
-			if stateObject.remove {
+		// flag the object as deleted.
-				stateObject.deleted = true
+		if stateObject.remove {
-			}
+			stateObject.deleted = true
-			stateObject.dirty = false
 		}
+		delete(s.stateObjectsDirty, addr)
 	}
 }
 
@@ -407,46 +516,40 @@ func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) {
 	return root, batch
 }
 
-func (s *StateDB) commit(db trie.DatabaseWriter) (common.Hash, error) {
+func (s *StateDB) commit(dbw trie.DatabaseWriter) (root common.Hash, err error) {
 	s.refund = new(big.Int)
 
-	for _, stateObject := range s.stateObjects {
+	// Commit objects to the trie.
+	for addr, stateObject := range s.stateObjects {
 		if stateObject.remove {
 			// If the object has been removed, don't bother syncing it
 			// and just mark it for deletion in the trie.
 			s.DeleteStateObject(stateObject)
-		} else {
+		} else if _, ok := s.stateObjectsDirty[addr]; ok {
 			// Write any contract code associated with the state object
-			if len(stateObject.code) > 0 {
+			if stateObject.code != nil && stateObject.dirtyCode {
-				if err := db.Put(stateObject.codeHash, stateObject.code); err != nil {
+				if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil {
 					return common.Hash{}, err
 				}
+				stateObject.dirtyCode = false
 			}
-			// Write any storage changes in the state object to its trie.
+			// Write any storage changes in the state object to its storage trie.
-			stateObject.Update()
+			if err := stateObject.CommitTrie(s.db, dbw); err != nil {
-
-			// Commit the trie of the object to the batch.
-			// This updates the trie root internally, so
-			// getting the root hash of the storage trie
-			// through UpdateStateObject is fast.
-			if _, err := stateObject.trie.CommitTo(db); err != nil {
 				return common.Hash{}, err
 			}
-			// Update the object in the account trie.
+			// Update the object in the main account trie.
 			s.UpdateStateObject(stateObject)
 		}
-		stateObject.dirty = false
+		delete(s.stateObjectsDirty, addr)
 	}
-	return s.trie.CommitTo(db)
+	// Write trie changes.
+	root, err = s.trie.CommitTo(dbw)
+	if err == nil {
+		s.pushTrie(s.trie)
+	}
+	return root, err
 }
 
 func (self *StateDB) Refunds() *big.Int {
 	return self.refund
 }
-
-// Debug stuff
-func (self *StateDB) CreateOutputForDiff() {
-	for _, stateObject := range self.stateObjects {
-		stateObject.CreateOutputForDiff()
-	}
-}

core/state/statedb_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
@@ -40,7 +41,7 @@ func TestUpdateLeaks(t *testing.T) {
 			obj.SetState(common.BytesToHash([]byte{i, i, i}), common.BytesToHash([]byte{i, i, i, i}))
 		}
 		if i%3 == 0 {
-			obj.SetCode([]byte{i, i, i, i, i})
+			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i})
 		}
 		state.UpdateStateObject(obj)
 	}
@@ -70,7 +71,7 @@ func TestIntermediateLeaks(t *testing.T) {
 			obj.SetState(common.BytesToHash([]byte{i, i, i, 0}), common.BytesToHash([]byte{i, i, i, i, 0}))
 		}
 		if i%3 == 0 {
-			obj.SetCode([]byte{i, i, i, i, i, 0})
+			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i, 0}), []byte{i, i, i, i, i, 0})
 		}
 		transState.UpdateStateObject(obj)
 
@@ -82,7 +83,7 @@ func TestIntermediateLeaks(t *testing.T) {
 			obj.SetState(common.BytesToHash([]byte{i, i, i, 1}), common.BytesToHash([]byte{i, i, i, i, 1}))
 		}
 		if i%3 == 0 {
-			obj.SetCode([]byte{i, i, i, i, i, 1})
+			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i, 1}), []byte{i, i, i, i, i, 1})
 		}
 		transState.UpdateStateObject(obj)
 
@@ -94,7 +95,7 @@ func TestIntermediateLeaks(t *testing.T) {
 			obj.SetState(common.BytesToHash([]byte{i, i, i, 1}), common.BytesToHash([]byte{i, i, i, i, 1}))
 		}
 		if i%3 == 0 {
-			obj.SetCode([]byte{i, i, i, i, i, 1})
+			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i, 1}), []byte{i, i, i, i, i, 1})
 		}
 		finalState.UpdateStateObject(obj)
 	}

core/state/sync_test.go

@@ -54,7 +54,7 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
 		acc.nonce = uint64(42 * i)
 
 		if i%3 == 0 {
-			obj.SetCode([]byte{i, i, i, i, i})
+			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i})
 			acc.code = []byte{i, i, i, i, i}
 		}
 		state.UpdateStateObject(obj)
@@ -62,9 +62,6 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
 	}
 	root, _ := state.Commit()
 
-	// Remove any potentially cached data from the test state creation
-	trie.ClearGlobalCache()
-
 	// Return the generated state
 	return db, root, accounts
 }
@@ -72,9 +69,6 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
 // checkStateAccounts cross references a reconstructed state with an expected
 // account array.
 func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
-	// Remove any potentially cached data from the state synchronisation
-	trie.ClearGlobalCache()
-
 	// Check root availability and state contents
 	state, err := New(root, db)
 	if err != nil {
@@ -98,9 +92,6 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou
 
 // checkStateConsistency checks that all nodes in a state trie are indeed present.
 func checkStateConsistency(db ethdb.Database, root common.Hash) error {
-	// Remove any potentially cached data from the test state creation or previous checks
-	trie.ClearGlobalCache()
-
 	// Create and iterate a state trie rooted in a sub-node
 	if _, err := db.Get(root.Bytes()); err != nil {
 		return nil // Consider a non existent state consistent

core/vm/contract.go

@@ -27,7 +27,7 @@ type ContractRef interface {
 	ReturnGas(*big.Int, *big.Int)
 	Address() common.Address
 	Value() *big.Int
-	SetCode([]byte)
+	SetCode(common.Hash, []byte)
 	ForEachStorage(callback func(key, value common.Hash) bool)
 }
 
@@ -44,8 +44,9 @@ type Contract struct {
 	jumpdests destinations // result of JUMPDEST analysis.
 
 	Code     []byte
-	Input    []byte
+	CodeHash common.Hash
 	CodeAddr *common.Address
+	Input    []byte
 
 	value, Gas, UsedGas, Price *big.Int
 
@@ -143,14 +144,16 @@ func (c *Contract) Value() *big.Int {
 }
 
 // SetCode sets the code to the contract
-func (self *Contract) SetCode(code []byte) {
+func (self *Contract) SetCode(hash common.Hash, code []byte) {
 	self.Code = code
+	self.CodeHash = hash
 }
 
 // SetCallCode sets the code of the contract and address of the backing data
 // object
-func (self *Contract) SetCallCode(addr *common.Address, code []byte) {
+func (self *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) {
 	self.Code = code
+	self.CodeHash = hash
 	self.CodeAddr = addr
 }
 

core/vm/environment.go

@@ -94,6 +94,8 @@ type Database interface {
 	GetNonce(common.Address) uint64
 	SetNonce(common.Address, uint64)
 
+	GetCodeHash(common.Address) common.Hash
+	GetCodeSize(common.Address) int
 	GetCode(common.Address) []byte
 	SetCode(common.Address, []byte)
 
@@ -117,7 +119,7 @@ type Account interface {
 	Balance() *big.Int
 	Address() common.Address
 	ReturnGas(*big.Int, *big.Int)
-	SetCode([]byte)
+	SetCode(common.Hash, []byte)
 	ForEachStorage(cb func(key, value common.Hash) bool)
 	Value() *big.Int
 }

core/vm/instructions.go

@@ -363,7 +363,7 @@ func opCalldataCopy(instr instruction, pc *uint64, env Environment, contract *Co
 
 func opExtCodeSize(instr instruction, pc *uint64, env Environment, contract *Contract, memory *Memory, stack *stack) {
 	addr := common.BigToAddress(stack.pop())
-	l := big.NewInt(int64(len(env.Db().GetCode(addr))))
+	l := big.NewInt(int64(env.Db().GetCodeSize(addr)))
 	stack.push(l)
 }
 

core/vm/jit_test.go

@@ -135,7 +135,7 @@ func (account) SetNonce(uint64)                                     {}
 func (account) Balance() *big.Int                                   { return nil }
 func (account) Address() common.Address                             { return common.Address{} }
 func (account) ReturnGas(*big.Int, *big.Int)                        {}
-func (account) SetCode([]byte)                                      {}
+func (account) SetCode(common.Hash, []byte)                         {}
 func (account) ForEachStorage(cb func(key, value common.Hash) bool) {}
 
 func runVmBench(test vmBench, b *testing.B) {

core/vm/logger_test.go

@@ -30,7 +30,7 @@ type dummyContractRef struct {
 func (dummyContractRef) ReturnGas(*big.Int, *big.Int) {}
 func (dummyContractRef) Address() common.Address      { return common.Address{} }
 func (dummyContractRef) Value() *big.Int              { return new(big.Int) }
-func (dummyContractRef) SetCode([]byte)               {}
+func (dummyContractRef) SetCode(common.Hash, []byte)  {}
 func (d *dummyContractRef) ForEachStorage(callback func(key, value common.Hash) bool) {
 	d.calledForEach = true
 }

core/vm/runtime/runtime.go

@@ -104,7 +104,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 		receiver = cfg.State.CreateAccount(common.StringToAddress("contract"))
 	)
 	// set the receiver's (the executing contract) code for execution.
-	receiver.SetCode(code)
+	receiver.SetCode(crypto.Keccak256Hash(code), code)
 
 	// Call the code with the given configuration.
 	ret, err := vmenv.Call(

core/vm/vm.go

@@ -79,10 +79,11 @@ func (evm *EVM) Run(contract *Contract, input []byte) (ret []byte, err error) {
 		return nil, nil
 	}
 
-	var (
+	codehash := contract.CodeHash // codehash is used when doing jump dest caching
-		codehash = crypto.Keccak256Hash(contract.Code) // codehash is used when doing jump dest caching
+	if codehash == (common.Hash{}) {
-		program  *Program
+		codehash = crypto.Keccak256Hash(contract.Code)
-	)
+	}
+	var program *Program
 	if evm.cfg.EnableJit {
 		// If the JIT is enabled check the status of the JIT program,
 		// if it doesn't exist compile a new program in a separate

eth/api.go

@@ -1575,14 +1575,14 @@ func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI {
 }
 
 // DumpBlock retrieves the entire state of the database at a given block.
-func (api *PublicDebugAPI) DumpBlock(number uint64) (state.World, error) {
+func (api *PublicDebugAPI) DumpBlock(number uint64) (state.Dump, error) {
 	block := api.eth.BlockChain().GetBlockByNumber(number)
 	if block == nil {
-		return state.World{}, fmt.Errorf("block #%d not found", number)
+		return state.Dump{}, fmt.Errorf("block #%d not found", number)
 	}
-	stateDb, err := state.New(block.Root(), api.eth.ChainDb())
+	stateDb, err := api.eth.BlockChain().StateAt(block.Root())
 	if err != nil {
-		return state.World{}, err
+		return state.Dump{}, err
 	}
 	return stateDb.RawDump(), nil
 }
@@ -1748,7 +1748,7 @@ func (api *PrivateDebugAPI) traceBlock(block *types.Block, config *vm.Config) (b
 	if err := core.ValidateHeader(api.config, blockchain.AuxValidator(), block.Header(), blockchain.GetHeader(block.ParentHash()), true, false); err != nil {
 		return false, collector.traces, err
 	}
-	statedb, err := state.New(blockchain.GetBlock(block.ParentHash()).Root(), api.eth.ChainDb())
+	statedb, err := blockchain.StateAt(blockchain.GetBlock(block.ParentHash()).Root())
 	if err != nil {
 		return false, collector.traces, err
 	}
@@ -1850,7 +1850,7 @@ func (api *PrivateDebugAPI) TraceTransaction(txHash common.Hash, logger *vm.LogC
 	if parent == nil {
 		return nil, fmt.Errorf("block parent %x not found", block.ParentHash())
 	}
-	stateDb, err := state.New(parent.Root(), api.eth.ChainDb())
+	stateDb, err := api.eth.BlockChain().StateAt(parent.Root())
 	if err != nil {
 		return nil, err
 	}

ethdb/database.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 
@@ -86,6 +87,7 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
 		OpenFilesCacheCapacity: handles,
 		BlockCacheCapacity:     cache / 2 * opt.MiB,
 		WriteBuffer:            cache / 4 * opt.MiB, // Two of these are used internally
+		Filter:                 filter.NewBloomFilter(10),
 	})
 	if _, corrupted := err.(*errors.ErrCorrupted); corrupted {
 		db, err = leveldb.RecoverFile(file, nil)

internal/build/env.go

@@ -0,0 +1,116 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+var (
+	// These flags override values in build env.
+	GitCommitFlag   = flag.String("git-commit", "", `Overrides git commit hash embedded into executables`)
+	GitBranchFlag   = flag.String("git-branch", "", `Overrides git branch being built`)
+	GitTagFlag      = flag.String("git-tag", "", `Overrides git tag being built`)
+	BuildnumFlag    = flag.String("buildnum", "", `Overrides CI build number`)
+	PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`)
+)
+
+// Environment contains metadata provided by the build environment.
+type Environment struct {
+	Name                string // name of the environment
+	Repo                string // name of GitHub repo
+	Commit, Branch, Tag string // Git info
+	Buildnum            string
+	IsPullRequest       bool
+}
+
+func (env Environment) String() string {
+	return fmt.Sprintf("%s env (commit:%s branch:%s tag:%s buildnum:%s pr:%t)",
+		env.Name, env.Commit, env.Branch, env.Tag, env.Buildnum, env.IsPullRequest)
+}
+
+// Env returns metadata about the current CI environment, falling back to LocalEnv
+// if not running on CI.
+func Env() Environment {
+	switch {
+	case os.Getenv("CI") == "true" && os.Getenv("TRAVIS") == "true":
+		return Environment{
+			Name:          "travis",
+			Repo:          os.Getenv("TRAVIS_REPO_SLUG"),
+			Commit:        os.Getenv("TRAVIS_COMMIT"),
+			Branch:        os.Getenv("TRAVIS_BRANCH"),
+			Tag:           os.Getenv("TRAVIS_TAG"),
+			Buildnum:      os.Getenv("TRAVIS_BUILD_NUMBER"),
+			IsPullRequest: os.Getenv("TRAVIS_PULL_REQUEST") != "false",
+		}
+	case os.Getenv("CI") == "True" && os.Getenv("APPVEYOR") == "True":
+		return Environment{
+			Name:          "appveyor",
+			Repo:          os.Getenv("APPVEYOR_REPO_NAME"),
+			Commit:        os.Getenv("APPVEYOR_REPO_COMMIT"),
+			Branch:        os.Getenv("APPVEYOR_REPO_BRANCH"),
+			Tag:           os.Getenv("APPVEYOR_REPO_TAG"),
+			Buildnum:      os.Getenv("APPVEYOR_BUILD_NUMBER"),
+			IsPullRequest: os.Getenv("APPVEYOR_PULL_REQUEST_NUMBER") != "",
+		}
+	default:
+		return LocalEnv()
+	}
+}
+
+// LocalEnv returns build environment metadata gathered from git.
+func LocalEnv() Environment {
+	env := applyEnvFlags(Environment{Name: "local", Repo: "ethereum/go-ethereum"})
+	if _, err := os.Stat(".git"); err != nil {
+		return env
+	}
+	if env.Commit == "" {
+		env.Commit = RunGit("rev-parse", "HEAD")
+	}
+	if env.Branch == "" {
+		if b := RunGit("rev-parse", "--abbrev-ref", "HEAD"); b != "HEAD" {
+			env.Branch = b
+		}
+	}
+	// Note that we don't get the current git tag. It would slow down
+	// builds and isn't used by anything.
+	return env
+}
+
+func applyEnvFlags(env Environment) Environment {
+	if !flag.Parsed() {
+		panic("you need to call flag.Parse before Env or LocalEnv")
+	}
+	if *GitCommitFlag != "" {
+		env.Commit = *GitCommitFlag
+	}
+	if *GitBranchFlag != "" {
+		env.Branch = *GitBranchFlag
+	}
+	if *GitTagFlag != "" {
+		env.Tag = *GitTagFlag
+	}
+	if *BuildnumFlag != "" {
+		env.Buildnum = *BuildnumFlag
+	}
+	if *PullRequestFlag {
+		env.IsPullRequest = true
+	}
+	return env
+}

internal/build/util.go

@@ -29,9 +29,7 @@ import (
 	"text/template"
 )
 
-var (
+var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
-	DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
-)
 
 // MustRun executes the given command and exits the host process for
 // any error.
@@ -69,6 +67,7 @@ func GOPATH() string {
 	return strings.Join(newpath, string(filepath.ListSeparator))
 }
 
+// VERSION returns the content of the VERSION file.
 func VERSION() string {
 	version, err := ioutil.ReadFile("VERSION")
 	if err != nil {
@@ -77,10 +76,8 @@ func VERSION() string {
 	return string(bytes.TrimSpace(version))
 }
 
-func GitCommit() string {
+// RunGit runs a git subcommand and returns its output.
-	return RunGit("rev-parse", "HEAD")
+// The command must complete successfully.
-}
-
 func RunGit(args ...string) string {
 	cmd := exec.Command("git", args...)
 	var stdout, stderr bytes.Buffer
@@ -94,12 +91,13 @@ func RunGit(args ...string) string {
 	return strings.TrimSpace(stdout.String())
 }
 
-// Render renders the given template file.
+// Render renders the given template file into outputFile.
 func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) {
 	tpl := template.Must(template.ParseFiles(templateFile))
 	render(tpl, outputFile, outputPerm, x)
 }
 
+// RenderString renders the given template string into outputFile.
 func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) {
 	tpl := template.Must(template.New("").Parse(templateContent))
 	render(tpl, outputFile, outputPerm, x)

light/state_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/trie"
 	"golang.org/x/net/context"
@@ -42,7 +43,6 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
 	case *TrieRequest:
 		t, _ := trie.New(req.root, odr.sdb)
 		req.proof = t.Prove(req.key)
-		trie.ClearGlobalCache()
 	case *NodeDataRequest:
 		req.data, _ = odr.sdb.Get(req.hash[:])
 	}
@@ -61,8 +61,8 @@ func makeTestState() (common.Hash, ethdb.Database) {
 			so.SetNonce(100)
 		}
 		so.AddBalance(big.NewInt(int64(i)))
-		so.SetCode([]byte{i, i, i})
+		so.SetCode(crypto.Keccak256Hash([]byte{i, i, i}), []byte{i, i, i})
-		so.Update()
+		so.UpdateRoot(sdb)
 		st.UpdateStateObject(so)
 	}
 	root, _ := st.Commit()
@@ -75,7 +75,6 @@ func TestLightStateOdr(t *testing.T) {
 	odr := &testOdr{sdb: sdb, ldb: ldb}
 	ls := NewLightState(root, odr)
 	ctx := context.Background()
-	trie.ClearGlobalCache()
 
 	for i := byte(0); i < 100; i++ {
 		addr := common.Address{i}
@@ -160,7 +159,6 @@ func TestLightStateSetCopy(t *testing.T) {
 	odr := &testOdr{sdb: sdb, ldb: ldb}
 	ls := NewLightState(root, odr)
 	ctx := context.Background()
-	trie.ClearGlobalCache()
 
 	for i := byte(0); i < 100; i++ {
 		addr := common.Address{i}
@@ -237,7 +235,6 @@ func TestLightStateDelete(t *testing.T) {
 	odr := &testOdr{sdb: sdb, ldb: ldb}
 	ls := NewLightState(root, odr)
 	ctx := context.Background()
-	trie.ClearGlobalCache()
 
 	addr := common.Address{42}
 

miner/worker.go

@@ -358,7 +358,7 @@ func (self *worker) push(work *Work) {
 
 // makeCurrent creates a new environment for the current cycle.
 func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error {
-	state, err := state.New(parent.Root(), self.eth.ChainDb())
+	state, err := self.chain.StateAt(parent.Root())
 	if err != nil {
 		return err
 	}

node/node_test.go

@@ -514,12 +514,32 @@ func TestAPIGather(t *testing.T) {
 	}{
 		"Zero APIs": {[]rpc.API{}, InstrumentedServiceMakerA},
 		"Single API": {[]rpc.API{
-			{"single", "1", &OneMethodApi{fun: func() { calls <- "single.v1" }}, true},
+			{
+				Namespace: "single",
+				Version:   "1",
+				Service:   &OneMethodApi{fun: func() { calls <- "single.v1" }},
+				Public:    true,
+			},
 		}, InstrumentedServiceMakerB},
 		"Many APIs": {[]rpc.API{
-			{"multi", "1", &OneMethodApi{fun: func() { calls <- "multi.v1" }}, true},
+			{
-			{"multi.v2", "2", &OneMethodApi{fun: func() { calls <- "multi.v2" }}, true},
+				Namespace: "multi",
-			{"multi.v2.nested", "2", &OneMethodApi{fun: func() { calls <- "multi.v2.nested" }}, true},
+				Version:   "1",
+				Service:   &OneMethodApi{fun: func() { calls <- "multi.v1" }},
+				Public:    true,
+			},
+			{
+				Namespace: "multi.v2",
+				Version:   "2",
+				Service:   &OneMethodApi{fun: func() { calls <- "multi.v2" }},
+				Public:    true,
+			},
+			{
+				Namespace: "multi.v2.nested",
+				Version:   "2",
+				Service:   &OneMethodApi{fun: func() { calls <- "multi.v2.nested" }},
+				Public:    true,
+			},
 		}, InstrumentedServiceMakerC},
 	}
 	for id, config := range services {

tests/block_test_util.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/logger/glog"
@@ -219,7 +220,7 @@ func (t *BlockTest) InsertPreState(db ethdb.Database) (*state.StateDB, error) {
 			return nil, err
 		}
 		obj := statedb.CreateAccount(common.HexToAddress(addrString))
-		obj.SetCode(code)
+		obj.SetCode(crypto.Keccak256Hash(code), code)
 		obj.SetBalance(balance)
 		obj.SetNonce(nonce)
 		for k, v := range acct.Storage {

tests/state_test_util.go

@@ -97,7 +97,7 @@ func benchStateTest(ruleSet RuleSet, test VmTest, env map[string]string, b *test
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, db)
 	for addr, account := range test.Pre {
-		obj := StateObjectFromAccount(db, addr, account)
+		obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty)
 		statedb.SetStateObject(obj)
 		for a, v := range account.Storage {
 			obj.SetState(common.HexToHash(a), common.HexToHash(v))
@@ -136,7 +136,7 @@ func runStateTest(ruleSet RuleSet, test VmTest) error {
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, db)
 	for addr, account := range test.Pre {
-		obj := StateObjectFromAccount(db, addr, account)
+		obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty)
 		statedb.SetStateObject(obj)
 		for a, v := range account.Storage {
 			obj.SetState(common.HexToHash(a), common.HexToHash(v))
@@ -187,7 +187,7 @@ func runStateTest(ruleSet RuleSet, test VmTest) error {
 		}
 
 		for addr, value := range account.Storage {
-			v := obj.GetState(common.HexToHash(addr))
+			v := statedb.GetState(obj.Address(), common.HexToHash(addr))
 			vexp := common.HexToHash(value)
 
 			if v != vexp {

tests/util.go

@@ -103,16 +103,18 @@ func (self Log) Topics() [][]byte {
 	return t
 }
 
-func StateObjectFromAccount(db ethdb.Database, addr string, account Account) *state.StateObject {
+func StateObjectFromAccount(db ethdb.Database, addr string, account Account, onDirty func(common.Address)) *state.StateObject {
-	obj := state.NewStateObject(common.HexToAddress(addr), db)
-	obj.SetBalance(common.Big(account.Balance))
-
 	if common.IsHex(account.Code) {
 		account.Code = account.Code[2:]
 	}
-	obj.SetCode(common.Hex2Bytes(account.Code))
+	code := common.Hex2Bytes(account.Code)
-	obj.SetNonce(common.Big(account.Nonce).Uint64())
+	codeHash := crypto.Keccak256Hash(code)
-
+	obj := state.NewObject(common.HexToAddress(addr), state.Account{
+		Balance:  common.Big(account.Balance),
+		CodeHash: codeHash[:],
+		Nonce:    common.Big(account.Nonce).Uint64(),
+	}, onDirty)
+	obj.SetCode(codeHash, code)
 	return obj
 }
 

tests/vm_test_util.go

@@ -103,7 +103,7 @@ func benchVmTest(test VmTest, env map[string]string, b *testing.B) {
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, db)
 	for addr, account := range test.Pre {
-		obj := StateObjectFromAccount(db, addr, account)
+		obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty)
 		statedb.SetStateObject(obj)
 		for a, v := range account.Storage {
 			obj.SetState(common.HexToHash(a), common.HexToHash(v))
@@ -154,7 +154,7 @@ func runVmTest(test VmTest) error {
 	db, _ := ethdb.NewMemDatabase()
 	statedb, _ := state.New(common.Hash{}, db)
 	for addr, account := range test.Pre {
-		obj := StateObjectFromAccount(db, addr, account)
+		obj := StateObjectFromAccount(db, addr, account, statedb.MarkStateObjectDirty)
 		statedb.SetStateObject(obj)
 		for a, v := range account.Storage {
 			obj.SetState(common.HexToHash(a), common.HexToHash(v))
@@ -205,11 +205,9 @@ func runVmTest(test VmTest) error {
 		if obj == nil {
 			continue
 		}
-
 		for addr, value := range account.Storage {
-			v := obj.GetState(common.HexToHash(addr))
+			v := statedb.GetState(obj.Address(), common.HexToHash(addr))
 			vexp := common.HexToHash(value)
-
 			if v != vexp {
 				return fmt.Errorf("(%x: %s) storage failed. Expected %x, got %x (%v %v)\n", obj.Address().Bytes()[0:4], addr, vexp, v, vexp.Big(), v.Big())
 			}

trie/arc.go

@@ -1,206 +0,0 @@
-// Copyright (c) 2015 Hans Alexander Gugel <alexander.gugel@gmail.com>
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-// This file contains a modified version of package arc from
-// https://github.com/alexanderGugel/arc
-//
-// It implements the ARC (Adaptive Replacement Cache) algorithm as detailed in
-// https://www.usenix.org/legacy/event/fast03/tech/full_papers/megiddo/megiddo.pdf
-
-package trie
-
-import (
-	"container/list"
-	"sync"
-)
-
-type arc struct {
-	p     int
-	c     int
-	t1    *list.List
-	b1    *list.List
-	t2    *list.List
-	b2    *list.List
-	cache map[string]*entry
-	mutex sync.Mutex
-}
-
-type entry struct {
-	key   hashNode
-	value node
-	ll    *list.List
-	el    *list.Element
-}
-
-// newARC returns a new Adaptive Replacement Cache with the
-// given capacity.
-func newARC(c int) *arc {
-	return &arc{
-		c:     c,
-		t1:    list.New(),
-		b1:    list.New(),
-		t2:    list.New(),
-		b2:    list.New(),
-		cache: make(map[string]*entry, c),
-	}
-}
-
-// Clear clears the cache
-func (a *arc) Clear() {
-	a.mutex.Lock()
-	defer a.mutex.Unlock()
-	a.p = 0
-	a.t1 = list.New()
-	a.b1 = list.New()
-	a.t2 = list.New()
-	a.b2 = list.New()
-	a.cache = make(map[string]*entry, a.c)
-}
-
-// Put inserts a new key-value pair into the cache.
-// This optimizes future access to this entry (side effect).
-func (a *arc) Put(key hashNode, value node) bool {
-	a.mutex.Lock()
-	defer a.mutex.Unlock()
-	ent, ok := a.cache[string(key)]
-	if ok != true {
-		ent = &entry{key: key, value: value}
-		a.req(ent)
-		a.cache[string(key)] = ent
-	} else {
-		ent.value = value
-		a.req(ent)
-	}
-	return ok
-}
-
-// Get retrieves a previously via Set inserted entry.
-// This optimizes future access to this entry (side effect).
-func (a *arc) Get(key hashNode) (value node, ok bool) {
-	a.mutex.Lock()
-	defer a.mutex.Unlock()
-	ent, ok := a.cache[string(key)]
-	if ok {
-		a.req(ent)
-		return ent.value, ent.value != nil
-	}
-	return nil, false
-}
-
-func (a *arc) req(ent *entry) {
-	if ent.ll == a.t1 || ent.ll == a.t2 {
-		// Case I
-		ent.setMRU(a.t2)
-	} else if ent.ll == a.b1 {
-		// Case II
-		// Cache Miss in t1 and t2
-
-		// Adaptation
-		var d int
-		if a.b1.Len() >= a.b2.Len() {
-			d = 1
-		} else {
-			d = a.b2.Len() / a.b1.Len()
-		}
-		a.p = a.p + d
-		if a.p > a.c {
-			a.p = a.c
-		}
-
-		a.replace(ent)
-		ent.setMRU(a.t2)
-	} else if ent.ll == a.b2 {
-		// Case III
-		// Cache Miss in t1 and t2
-
-		// Adaptation
-		var d int
-		if a.b2.Len() >= a.b1.Len() {
-			d = 1
-		} else {
-			d = a.b1.Len() / a.b2.Len()
-		}
-		a.p = a.p - d
-		if a.p < 0 {
-			a.p = 0
-		}
-
-		a.replace(ent)
-		ent.setMRU(a.t2)
-	} else if ent.ll == nil {
-		// Case IV
-
-		if a.t1.Len()+a.b1.Len() == a.c {
-			// Case A
-			if a.t1.Len() < a.c {
-				a.delLRU(a.b1)
-				a.replace(ent)
-			} else {
-				a.delLRU(a.t1)
-			}
-		} else if a.t1.Len()+a.b1.Len() < a.c {
-			// Case B
-			if a.t1.Len()+a.t2.Len()+a.b1.Len()+a.b2.Len() >= a.c {
-				if a.t1.Len()+a.t2.Len()+a.b1.Len()+a.b2.Len() == 2*a.c {
-					a.delLRU(a.b2)
-				}
-				a.replace(ent)
-			}
-		}
-
-		ent.setMRU(a.t1)
-	}
-}
-
-func (a *arc) delLRU(list *list.List) {
-	lru := list.Back()
-	list.Remove(lru)
-	delete(a.cache, string(lru.Value.(*entry).key))
-}
-
-func (a *arc) replace(ent *entry) {
-	if a.t1.Len() > 0 && ((a.t1.Len() > a.p) || (ent.ll == a.b2 && a.t1.Len() == a.p)) {
-		lru := a.t1.Back().Value.(*entry)
-		lru.value = nil
-		lru.setMRU(a.b1)
-	} else {
-		lru := a.t2.Back().Value.(*entry)
-		lru.value = nil
-		lru.setMRU(a.b2)
-	}
-}
-
-func (e *entry) setLRU(list *list.List) {
-	e.detach()
-	e.ll = list
-	e.el = e.ll.PushBack(e)
-}
-
-func (e *entry) setMRU(list *list.List) {
-	e.detach()
-	e.ll = list
-	e.el = e.ll.PushFront(e)
-}
-
-func (e *entry) detach() {
-	if e.ll != nil {
-		e.ll.Remove(e.el)
-	}
-}

trie/hasher.go

@@ -0,0 +1,157 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"bytes"
+	"hash"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+type hasher struct {
+	tmp *bytes.Buffer
+	sha hash.Hash
+}
+
+// hashers live in a global pool.
+var hasherPool = sync.Pool{
+	New: func() interface{} {
+		return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()}
+	},
+}
+
+func newHasher() *hasher {
+	return hasherPool.Get().(*hasher)
+}
+
+func returnHasherToPool(h *hasher) {
+	hasherPool.Put(h)
+}
+
+// hash collapses a node down into a hash node, also returning a copy of the
+// original node initialzied with the computed hash to replace the original one.
+func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) {
+	// If we're not storing the node, just hashing, use avaialble cached data
+	if hash, dirty := n.cache(); hash != nil && (db == nil || !dirty) {
+		return hash, n, nil
+	}
+	// Trie not processed yet or needs storage, walk the children
+	collapsed, cached, err := h.hashChildren(n, db)
+	if err != nil {
+		return hashNode{}, n, err
+	}
+	hashed, err := h.store(collapsed, db, force)
+	if err != nil {
+		return hashNode{}, n, err
+	}
+	// Cache the hash and RLP blob of the ndoe for later reuse
+	if hash, ok := hashed.(hashNode); ok && !force {
+		switch cached := cached.(type) {
+		case shortNode:
+			cached.hash = hash
+			if db != nil {
+				cached.dirty = false
+			}
+			return hashed, cached, nil
+		case fullNode:
+			cached.hash = hash
+			if db != nil {
+				cached.dirty = false
+			}
+			return hashed, cached, nil
+		}
+	}
+	return hashed, cached, nil
+}
+
+// hashChildren replaces the children of a node with their hashes if the encoded
+// size of the child is larger than a hash, returning the collapsed node as well
+// as a replacement for the original node with the child hashes cached in.
+func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, error) {
+	var err error
+
+	switch n := original.(type) {
+	case shortNode:
+		// Hash the short node's child, caching the newly hashed subtree
+		cached := n
+		cached.Key = common.CopyBytes(cached.Key)
+
+		n.Key = compactEncode(n.Key)
+		if _, ok := n.Val.(valueNode); !ok {
+			if n.Val, cached.Val, err = h.hash(n.Val, db, false); err != nil {
+				return n, original, err
+			}
+		}
+		if n.Val == nil {
+			n.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
+		}
+		return n, cached, nil
+
+	case fullNode:
+		// Hash the full node's children, caching the newly hashed subtrees
+		cached := fullNode{dirty: n.dirty}
+
+		for i := 0; i < 16; i++ {
+			if n.Children[i] != nil {
+				if n.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false); err != nil {
+					return n, original, err
+				}
+			} else {
+				n.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
+			}
+		}
+		cached.Children[16] = n.Children[16]
+		if n.Children[16] == nil {
+			n.Children[16] = valueNode(nil)
+		}
+		return n, cached, nil
+
+	default:
+		// Value and hash nodes don't have children so they're left as were
+		return n, original, nil
+	}
+}
+
+func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
+	// Don't store hashes or empty nodes.
+	if _, isHash := n.(hashNode); n == nil || isHash {
+		return n, nil
+	}
+	// Generate the RLP encoding of the node
+	h.tmp.Reset()
+	if err := rlp.Encode(h.tmp, n); err != nil {
+		panic("encode error: " + err.Error())
+	}
+	if h.tmp.Len() < 32 && !force {
+		return n, nil // Nodes smaller than 32 bytes are stored inside their parent
+	}
+	// Larger nodes are replaced by their hash and stored in the database.
+	hash, _ := n.cache()
+	if hash == nil {
+		h.sha.Reset()
+		h.sha.Write(h.tmp.Bytes())
+		hash = hashNode(h.sha.Sum(nil))
+	}
+	if db != nil {
+		return hash, db.Put(hash, h.tmp.Bytes())
+	}
+	return hash, nil
+}

trie/iterator.go

@@ -16,18 +16,13 @@
 
 package trie
 
-import (
+import "github.com/ethereum/go-ethereum/common"
-	"bytes"
-	"fmt"
 
-	"github.com/ethereum/go-ethereum/common"
+// Iterator is a key-value trie iterator that traverses a Trie.
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
-)
-
-// Iterator is a key-value trie iterator to traverse the data contents.
 type Iterator struct {
-	trie *Trie
+	trie   *Trie
+	nodeIt *NodeIterator
+	keyBuf []byte
 
 	Key   []byte // Current data key on which the iterator is positioned on
 	Value []byte // Current data value on which the iterator is positioned on
@@ -35,119 +30,45 @@ type Iterator struct {
 
 // NewIterator creates a new key-value iterator.
 func NewIterator(trie *Trie) *Iterator {
-	return &Iterator{trie: trie, Key: nil}
+	return &Iterator{
+		trie:   trie,
+		nodeIt: NewNodeIterator(trie),
+		keyBuf: make([]byte, 0, 64),
+		Key:    nil,
+	}
 }
 
-// Next moves the iterator forward with one key-value entry.
+// Next moves the iterator forward one key-value entry.
-func (self *Iterator) Next() bool {
+func (it *Iterator) Next() bool {
-	isIterStart := false
+	for it.nodeIt.Next() {
-	if self.Key == nil {
+		if it.nodeIt.Leaf {
-		isIterStart = true
+			it.Key = it.makeKey()
-		self.Key = make([]byte, 32)
+			it.Value = it.nodeIt.LeafBlob
+			return true
+		}
 	}
-
+	it.Key = nil
-	key := remTerm(compactHexDecode(self.Key))
+	it.Value = nil
-	k := self.next(self.trie.root, key, isIterStart)
+	return false
-
-	self.Key = []byte(decodeCompact(k))
-
-	return len(k) > 0
 }
 
-func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byte {
+func (it *Iterator) makeKey() []byte {
-	if node == nil {
+	key := it.keyBuf[:0]
-		return nil
+	for _, se := range it.nodeIt.stack {
+		switch node := se.node.(type) {
+		case fullNode:
+			if se.child <= 16 {
+				key = append(key, byte(se.child))
+			}
+		case shortNode:
+			if hasTerm(node.Key) {
+				key = append(key, node.Key[:len(node.Key)-1]...)
+			} else {
+				key = append(key, node.Key...)
+			}
+		}
 	}
-
+	return decodeCompact(key)
-	switch node := node.(type) {
-	case fullNode:
-		if len(key) > 0 {
-			k := self.next(node.Children[key[0]], key[1:], isIterStart)
-			if k != nil {
-				return append([]byte{key[0]}, k...)
-			}
-		}
-
-		var r byte
-		if len(key) > 0 {
-			r = key[0] + 1
-		}
-
-		for i := r; i < 16; i++ {
-			k := self.key(node.Children[i])
-			if k != nil {
-				return append([]byte{i}, k...)
-			}
-		}
-
-	case shortNode:
-		k := remTerm(node.Key)
-		if vnode, ok := node.Val.(valueNode); ok {
-			switch bytes.Compare([]byte(k), key) {
-			case 0:
-				if isIterStart {
-					self.Value = vnode
-					return k
-				}
-			case 1:
-				self.Value = vnode
-				return k
-			}
-		} else {
-			cnode := node.Val
-
-			var ret []byte
-			skey := key[len(k):]
-			if bytes.HasPrefix(key, k) {
-				ret = self.next(cnode, skey, isIterStart)
-			} else if bytes.Compare(k, key[:len(k)]) > 0 {
-				return self.key(node)
-			}
-
-			if ret != nil {
-				return append(k, ret...)
-			}
-		}
-
-	case hashNode:
-		rn, err := self.trie.resolveHash(node, nil, nil)
-		if err != nil && glog.V(logger.Error) {
-			glog.Errorf("Unhandled trie error: %v", err)
-		}
-		return self.next(rn, key, isIterStart)
-	}
-	return nil
-}
-
-func (self *Iterator) key(node interface{}) []byte {
-	switch node := node.(type) {
-	case shortNode:
-		// Leaf node
-		k := remTerm(node.Key)
-		if vnode, ok := node.Val.(valueNode); ok {
-			self.Value = vnode
-			return k
-		}
-		return append(k, self.key(node.Val)...)
-	case fullNode:
-		if node.Children[16] != nil {
-			self.Value = node.Children[16].(valueNode)
-			return []byte{16}
-		}
-		for i := 0; i < 16; i++ {
-			k := self.key(node.Children[i])
-			if k != nil {
-				return append([]byte{byte(i)}, k...)
-			}
-		}
-	case hashNode:
-		rn, err := self.trie.resolveHash(node, nil, nil)
-		if err != nil && glog.V(logger.Error) {
-			glog.Errorf("Unhandled trie error: %v", err)
-		}
-		return self.key(rn)
-	}
-	return nil
 }
 
 // nodeIteratorState represents the iteration state at one particular node of the
@@ -199,25 +120,27 @@ func (it *NodeIterator) Next() bool {
 
 // step moves the iterator to the next node of the trie.
 func (it *NodeIterator) step() error {
-	// Abort if we reached the end of the iteration
 	if it.trie == nil {
+		// Abort if we reached the end of the iteration
 		return nil
 	}
-	// Initialize the iterator if we've just started, or pop off the old node otherwise
 	if len(it.stack) == 0 {
-		// Always start with a collapsed root
+		// Initialize the iterator if we've just started.
 		root := it.trie.Hash()
-		it.stack = append(it.stack, &nodeIteratorState{node: hashNode(root[:]), child: -1})
+		state := &nodeIteratorState{node: it.trie.root, child: -1}
-		if it.stack[0].node == nil {
+		if root != emptyRoot {
-			return fmt.Errorf("root node missing: %x", it.trie.Hash())
+			state.hash = root
 		}
+		it.stack = append(it.stack, state)
 	} else {
+		// Continue iterating at the previous node otherwise.
 		it.stack = it.stack[:len(it.stack)-1]
 		if len(it.stack) == 0 {
 			it.trie = nil
 			return nil
 		}
 	}
+
 	// Continue iteration to the next child
 	for {
 		parent := it.stack[len(it.stack)-1]
@@ -232,7 +155,12 @@ func (it *NodeIterator) step() error {
 			}
 			for parent.child++; parent.child < len(node.Children); parent.child++ {
 				if current := node.Children[parent.child]; current != nil {
-					it.stack = append(it.stack, &nodeIteratorState{node: current, parent: ancestor, child: -1})
+					it.stack = append(it.stack, &nodeIteratorState{
+						hash:   common.BytesToHash(node.hash),
+						node:   current,
+						parent: ancestor,
+						child:  -1,
+					})
 					break
 				}
 			}
@@ -242,7 +170,12 @@ func (it *NodeIterator) step() error {
 				break
 			}
 			parent.child++
-			it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1})
+			it.stack = append(it.stack, &nodeIteratorState{
+				hash:   common.BytesToHash(node.hash),
+				node:   node.Val,
+				parent: ancestor,
+				child:  -1,
+			})
 		} else if hash, ok := parent.node.(hashNode); ok {
 			// Hash node, resolve the hash child from the database, then the node itself
 			if parent.child >= 0 {
@@ -254,7 +187,12 @@ func (it *NodeIterator) step() error {
 			if err != nil {
 				return err
 			}
-			it.stack = append(it.stack, &nodeIteratorState{hash: common.BytesToHash(hash), node: node, parent: ancestor, child: -1})
+			it.stack = append(it.stack, &nodeIteratorState{
+				hash:   common.BytesToHash(hash),
+				node:   node,
+				parent: ancestor,
+				child:  -1,
+			})
 		} else {
 			break
 		}

trie/iterator_test.go

@@ -34,21 +34,60 @@ func TestIterator(t *testing.T) {
 		{"dog", "puppy"},
 		{"somethingveryoddindeedthis is", "myothernodedata"},
 	}
-	v := make(map[string]bool)
+	all := make(map[string]string)
 	for _, val := range vals {
-		v[val.k] = false
+		all[val.k] = val.v
 		trie.Update([]byte(val.k), []byte(val.v))
 	}
 	trie.Commit()
 
+	found := make(map[string]string)
 	it := NewIterator(trie)
 	for it.Next() {
-		v[string(it.Key)] = true
+		found[string(it.Key)] = string(it.Value)
 	}
 
-	for k, found := range v {
+	for k, v := range all {
-		if !found {
+		if found[k] != v {
-			t.Error("iterator didn't find", k)
+			t.Errorf("iterator value mismatch for %s: got %q want %q", k, found[k], v)
+		}
+	}
+}
+
+type kv struct {
+	k, v []byte
+	t    bool
+}
+
+func TestIteratorLargeData(t *testing.T) {
+	trie := newEmpty()
+	vals := make(map[string]*kv)
+
+	for i := byte(0); i < 255; i++ {
+		value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
+		value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false}
+		trie.Update(value.k, value.v)
+		trie.Update(value2.k, value2.v)
+		vals[string(value.k)] = value
+		vals[string(value2.k)] = value2
+	}
+
+	it := NewIterator(trie)
+	for it.Next() {
+		vals[string(it.Key)].t = true
+	}
+
+	var untouched []*kv
+	for _, value := range vals {
+		if !value.t {
+			untouched = append(untouched, value)
+		}
+	}
+
+	if len(untouched) > 0 {
+		t.Errorf("Missed %d nodes", len(untouched))
+		for _, value := range untouched {
+			t.Error(value)
 		}
 	}
 }

trie/proof.go

@@ -70,15 +70,13 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue {
 			panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
 		}
 	}
-	if t.hasher == nil {
+	hasher := newHasher()
-		t.hasher = newHasher()
-	}
 	proof := make([]rlp.RawValue, 0, len(nodes))
 	for i, n := range nodes {
 		// Don't bother checking for errors here since hasher panics
 		// if encoding doesn't work and we're not writing to any database.
-		n, _, _ = t.hasher.hashChildren(n, nil)
+		n, _, _ = hasher.hashChildren(n, nil)
-		hn, _ := t.hasher.store(n, nil, false)
+		hn, _ := hasher.store(n, nil, false)
 		if _, ok := hn.(hashNode); ok || i == 0 {
 			// If the node's database encoding is a hash (or is the
 			// root node), it becomes a proof element.

trie/secure_trie.go

@@ -17,16 +17,15 @@
 package trie
 
 import (
-	"hash"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 )
 
 var secureKeyPrefix = []byte("secure-key-")
 
+const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
+
 // SecureTrie wraps a trie with key hashing. In a secure trie, all
 // access operations hash the key using keccak256. This prevents
 // calling code from creating long chains of nodes that
@@ -38,12 +37,11 @@ var secureKeyPrefix = []byte("secure-key-")
 //
 // SecureTrie is not safe for concurrent use.
 type SecureTrie struct {
-	*Trie
+	trie             Trie
-
+	hashKeyBuf       [secureKeyLength]byte
-	hash        hash.Hash
+	secKeyBuf        [200]byte
-	hashKeyBuf  []byte
+	secKeyCache      map[string][]byte
-	secKeyBuf   []byte
+	secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
-	secKeyCache map[string][]byte
 }
 
 // NewSecure creates a trie with an existing root node from db.
@@ -61,8 +59,7 @@ func NewSecure(root common.Hash, db Database) (*SecureTrie, error) {
 		return nil, err
 	}
 	return &SecureTrie{
-		Trie:        trie,
+		trie: *trie,
-		secKeyCache: make(map[string][]byte),
 	}, nil
 }
 
@@ -80,7 +77,7 @@ func (t *SecureTrie) Get(key []byte) []byte {
 // The value bytes must not be modified by the caller.
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *SecureTrie) TryGet(key []byte) ([]byte, error) {
-	return t.Trie.TryGet(t.hashKey(key))
+	return t.trie.TryGet(t.hashKey(key))
 }
 
 // Update associates key with value in the trie. Subsequent calls to
@@ -105,11 +102,11 @@ func (t *SecureTrie) Update(key, value []byte) {
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *SecureTrie) TryUpdate(key, value []byte) error {
 	hk := t.hashKey(key)
-	err := t.Trie.TryUpdate(hk, value)
+	err := t.trie.TryUpdate(hk, value)
 	if err != nil {
 		return err
 	}
-	t.secKeyCache[string(hk)] = common.CopyBytes(key)
+	t.getSecKeyCache()[string(hk)] = common.CopyBytes(key)
 	return nil
 }
 
@@ -124,17 +121,17 @@ func (t *SecureTrie) Delete(key []byte) {
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *SecureTrie) TryDelete(key []byte) error {
 	hk := t.hashKey(key)
-	delete(t.secKeyCache, string(hk))
+	delete(t.getSecKeyCache(), string(hk))
-	return t.Trie.TryDelete(hk)
+	return t.trie.TryDelete(hk)
 }
 
 // GetKey returns the sha3 preimage of a hashed key that was
 // previously used to store a value.
 func (t *SecureTrie) GetKey(shaKey []byte) []byte {
-	if key, ok := t.secKeyCache[string(shaKey)]; ok {
+	if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
 		return key
 	}
-	key, _ := t.Trie.db.Get(t.secKey(shaKey))
+	key, _ := t.trie.db.Get(t.secKey(shaKey))
 	return key
 }
 
@@ -144,7 +141,23 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
 // Committing flushes nodes from memory. Subsequent Get calls will load nodes
 // from the database.
 func (t *SecureTrie) Commit() (root common.Hash, err error) {
-	return t.CommitTo(t.db)
+	return t.CommitTo(t.trie.db)
+}
+
+func (t *SecureTrie) Hash() common.Hash {
+	return t.trie.Hash()
+}
+
+func (t *SecureTrie) Root() []byte {
+	return t.trie.Root()
+}
+
+func (t *SecureTrie) Iterator() *Iterator {
+	return t.trie.Iterator()
+}
+
+func (t *SecureTrie) NodeIterator() *NodeIterator {
+	return NewNodeIterator(&t.trie)
 }
 
 // CommitTo writes all nodes and the secure hash pre-images to the given database.
@@ -154,7 +167,7 @@ func (t *SecureTrie) Commit() (root common.Hash, err error) {
 // the trie's database. Calling code must ensure that the changes made to db are
 // written back to the trie's attached database before using the trie.
 func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
-	if len(t.secKeyCache) > 0 {
+	if len(t.getSecKeyCache()) > 0 {
 		for hk, key := range t.secKeyCache {
 			if err := db.Put(t.secKey([]byte(hk)), key); err != nil {
 				return common.Hash{}, err
@@ -162,27 +175,37 @@ func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
 		}
 		t.secKeyCache = make(map[string][]byte)
 	}
-	n, clean, err := t.hashRoot(db)
+	return t.trie.CommitTo(db)
-	if err != nil {
-		return (common.Hash{}), err
-	}
-	t.root = clean
-	return common.BytesToHash(n.(hashNode)), nil
 }
 
+// secKey returns the database key for the preimage of key, as an ephemeral buffer.
+// The caller must not hold onto the return value because it will become
+// invalid on the next call to hashKey or secKey.
 func (t *SecureTrie) secKey(key []byte) []byte {
-	t.secKeyBuf = append(t.secKeyBuf[:0], secureKeyPrefix...)
+	buf := append(t.secKeyBuf[:0], secureKeyPrefix...)
-	t.secKeyBuf = append(t.secKeyBuf, key...)
+	buf = append(buf, key...)
-	return t.secKeyBuf
+	return buf
 }
 
+// hashKey returns the hash of key as an ephemeral buffer.
+// The caller must not hold onto the return value because it will become
+// invalid on the next call to hashKey or secKey.
 func (t *SecureTrie) hashKey(key []byte) []byte {
-	if t.hash == nil {
+	h := newHasher()
-		t.hash = sha3.NewKeccak256()
+	h.sha.Reset()
-		t.hashKeyBuf = make([]byte, 32)
+	h.sha.Write(key)
-	}
+	buf := h.sha.Sum(t.hashKeyBuf[:0])
-	t.hash.Reset()
+	returnHasherToPool(h)
-	t.hash.Write(key)
+	return buf
-	t.hashKeyBuf = t.hash.Sum(t.hashKeyBuf[:0])
+}
-	return t.hashKeyBuf
+
+// getSecKeyCache returns the current secure key cache, creating a new one if
+// ownership changed (i.e. the current secure trie is a copy of another owning
+// the actual cache).
+func (t *SecureTrie) getSecKeyCache() map[string][]byte {
+	if t != t.secKeyCacheOwner {
+		t.secKeyCacheOwner = t
+		t.secKeyCache = make(map[string][]byte)
+	}
+	return t.secKeyCache
 }

trie/secure_trie_test.go

@@ -18,6 +18,8 @@ package trie
 
 import (
 	"bytes"
+	"runtime"
+	"sync"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -31,6 +33,37 @@ func newEmptySecure() *SecureTrie {
 	return trie
 }
 
+// makeTestSecureTrie creates a large enough secure trie for testing.
+func makeTestSecureTrie() (ethdb.Database, *SecureTrie, map[string][]byte) {
+	// Create an empty trie
+	db, _ := ethdb.NewMemDatabase()
+	trie, _ := NewSecure(common.Hash{}, db)
+
+	// Fill it with some arbitrary data
+	content := make(map[string][]byte)
+	for i := byte(0); i < 255; i++ {
+		// Map the same data under multiple keys
+		key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i}
+		content[string(key)] = val
+		trie.Update(key, val)
+
+		key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i}
+		content[string(key)] = val
+		trie.Update(key, val)
+
+		// Add some other data to inflate th trie
+		for j := byte(3); j < 13; j++ {
+			key, val = common.LeftPadBytes([]byte{j, i}, 32), []byte{j, i}
+			content[string(key)] = val
+			trie.Update(key, val)
+		}
+	}
+	trie.Commit()
+
+	// Return the generated trie
+	return db, trie, content
+}
+
 func TestSecureDelete(t *testing.T) {
 	trie := newEmptySecure()
 	vals := []struct{ k, v string }{
@@ -72,3 +105,41 @@ func TestSecureGetKey(t *testing.T) {
 		t.Errorf("GetKey returned %q, want %q", k, key)
 	}
 }
+
+func TestSecureTrieConcurrency(t *testing.T) {
+	// Create an initial trie and copy if for concurrent access
+	_, trie, _ := makeTestSecureTrie()
+
+	threads := runtime.NumCPU()
+	tries := make([]*SecureTrie, threads)
+	for i := 0; i < threads; i++ {
+		cpy := *trie
+		tries[i] = &cpy
+	}
+	// Start a batch of goroutines interactng with the trie
+	pend := new(sync.WaitGroup)
+	pend.Add(threads)
+	for i := 0; i < threads; i++ {
+		go func(index int) {
+			defer pend.Done()
+
+			for j := byte(0); j < 255; j++ {
+				// Map the same data under multiple keys
+				key, val := common.LeftPadBytes([]byte{byte(index), 1, j}, 32), []byte{j}
+				tries[index].Update(key, val)
+
+				key, val = common.LeftPadBytes([]byte{byte(index), 2, j}, 32), []byte{j}
+				tries[index].Update(key, val)
+
+				// Add some other data to inflate the trie
+				for k := byte(3); k < 13; k++ {
+					key, val = common.LeftPadBytes([]byte{byte(index), k, j}, 32), []byte{k, j}
+					tries[index].Update(key, val)
+				}
+			}
+			tries[index].Commit()
+		}(i)
+	}
+	// Wait for all threads to finish
+	pend.Wait()
+}

trie/sync_test.go

@@ -51,9 +51,6 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
 	}
 	trie.Commit()
 
-	// Remove any potentially cached data from the test trie creation
-	globalCache.Clear()
-
 	// Return the generated trie
 	return db, trie, content
 }
@@ -61,9 +58,6 @@ func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
 // checkTrieContents cross references a reconstructed trie with an expected data
 // content map.
 func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) {
-	// Remove any potentially cached data from the trie synchronisation
-	globalCache.Clear()
-
 	// Check root availability and trie contents
 	trie, err := New(common.BytesToHash(root), db)
 	if err != nil {
@@ -81,9 +75,6 @@ func checkTrieContents(t *testing.T, db Database, root []byte, content map[strin
 
 // checkTrieConsistency checks that all nodes in a trie are indeed present.
 func checkTrieConsistency(db Database, root common.Hash) error {
-	// Remove any potentially cached data from the test trie creation or previous checks
-	globalCache.Clear()
-
 	// Create and iterate a trie rooted in a subnode
 	trie, err := New(root, db)
 	if err != nil {

trie/trie.go

@@ -20,22 +20,14 @@ package trie
 import (
 	"bytes"
 	"fmt"
-	"hash"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
-	"github.com/ethereum/go-ethereum/rlp"
 )
 
-const defaultCacheCapacity = 800
-
 var (
-	// The global cache stores decoded trie nodes by hash as they get loaded.
-	globalCache = newARC(defaultCacheCapacity)
-
 	// This is the known root hash of an empty trie.
 	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
 
@@ -43,11 +35,6 @@ var (
 	emptyState = crypto.Keccak256Hash(nil)
 )
 
-// ClearGlobalCache clears the global trie cache
-func ClearGlobalCache() {
-	globalCache.Clear()
-}
-
 // Database must be implemented by backing stores for the trie.
 type Database interface {
 	DatabaseWriter
@@ -72,7 +59,6 @@ type Trie struct {
 	root         node
 	db           Database
 	originalRoot common.Hash
-	*hasher
 }
 
 // New creates a trie with an existing root node from db.
@@ -118,32 +104,50 @@ func (t *Trie) Get(key []byte) []byte {
 // If a node was not found in the database, a MissingNodeError is returned.
 func (t *Trie) TryGet(key []byte) ([]byte, error) {
 	key = compactHexDecode(key)
-	pos := 0
+	value, newroot, didResolve, err := t.tryGet(t.root, key, 0)
-	tn := t.root
+	if err == nil && didResolve {
-	for pos < len(key) {
+		t.root = newroot
-		switch n := tn.(type) {
+	}
-		case shortNode:
+	return value, err
-			if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) {
+}
-				return nil, nil
+
-			}
+func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode node, didResolve bool, err error) {
-			tn = n.Val
+	switch n := (origNode).(type) {
-			pos += len(n.Key)
+	case nil:
-		case fullNode:
+		return nil, nil, false, nil
-			tn = n.Children[key[pos]]
+	case valueNode:
-			pos++
+		return n, n, false, nil
-		case nil:
+	case shortNode:
-			return nil, nil
+		if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) {
-		case hashNode:
+			// key not found in trie
-			var err error
+			return nil, n, false, nil
-			tn, err = t.resolveHash(n, key[:pos], key[pos:])
+		}
-			if err != nil {
+		value, newnode, didResolve, err = t.tryGet(n.Val, key, pos+len(n.Key))
-				return nil, err
+		if err == nil && didResolve {
-			}
+			n.Val = newnode
-		default:
+			return value, n, didResolve, err
-			panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
+		} else {
-		}
+			return value, origNode, didResolve, err
+		}
+	case fullNode:
+		child := n.Children[key[pos]]
+		value, newnode, didResolve, err = t.tryGet(child, key, pos+1)
+		if err == nil && didResolve {
+			n.Children[key[pos]] = newnode
+			return value, n, didResolve, err
+		} else {
+			return value, origNode, didResolve, err
+		}
+	case hashNode:
+		child, err := t.resolveHash(n, key[:pos], key[pos:])
+		if err != nil {
+			return nil, n, true, err
+		}
+		value, newnode, _, err := t.tryGet(child, key, pos)
+		return value, newnode, true, err
+	default:
+		panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
 	}
-	return tn.(valueNode), nil
 }
 
 // Update associates key with value in the trie. Subsequent calls to
@@ -370,6 +374,9 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 		// n still contains at least two values and cannot be reduced.
 		return true, n, nil
 
+	case valueNode:
+		return true, nil, nil
+
 	case nil:
 		return false, nil, nil
 
@@ -410,9 +417,6 @@ func (t *Trie) resolve(n node, prefix, suffix []byte) (node, error) {
 }
 
 func (t *Trie) resolveHash(n hashNode, prefix, suffix []byte) (node, error) {
-	if v, ok := globalCache.Get(n); ok {
-		return v, nil
-	}
 	enc, err := t.db.Get(n)
 	if err != nil || enc == nil {
 		return nil, &MissingNodeError{
@@ -424,9 +428,6 @@ func (t *Trie) resolveHash(n hashNode, prefix, suffix []byte) (node, error) {
 		}
 	}
 	dec := mustDecodeNode(n, enc)
-	if dec != nil {
-		globalCache.Put(n, dec)
-	}
 	return dec, nil
 }
 
@@ -474,127 +475,7 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) {
 	if t.root == nil {
 		return hashNode(emptyRoot.Bytes()), nil, nil
 	}
-	if t.hasher == nil {
+	h := newHasher()
-		t.hasher = newHasher()
+	defer returnHasherToPool(h)
-	}
+	return h.hash(t.root, db, true)
-	return t.hasher.hash(t.root, db, true)
-}
-
-type hasher struct {
-	tmp *bytes.Buffer
-	sha hash.Hash
-}
-
-func newHasher() *hasher {
-	return &hasher{tmp: new(bytes.Buffer), sha: sha3.NewKeccak256()}
-}
-
-// hash collapses a node down into a hash node, also returning a copy of the
-// original node initialzied with the computed hash to replace the original one.
-func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) {
-	// If we're not storing the node, just hashing, use avaialble cached data
-	if hash, dirty := n.cache(); hash != nil && (db == nil || !dirty) {
-		return hash, n, nil
-	}
-	// Trie not processed yet or needs storage, walk the children
-	collapsed, cached, err := h.hashChildren(n, db)
-	if err != nil {
-		return hashNode{}, n, err
-	}
-	hashed, err := h.store(collapsed, db, force)
-	if err != nil {
-		return hashNode{}, n, err
-	}
-	// Cache the hash and RLP blob of the ndoe for later reuse
-	if hash, ok := hashed.(hashNode); ok && !force {
-		switch cached := cached.(type) {
-		case shortNode:
-			cached.hash = hash
-			if db != nil {
-				cached.dirty = false
-			}
-			return hashed, cached, nil
-		case fullNode:
-			cached.hash = hash
-			if db != nil {
-				cached.dirty = false
-			}
-			return hashed, cached, nil
-		}
-	}
-	return hashed, cached, nil
-}
-
-// hashChildren replaces the children of a node with their hashes if the encoded
-// size of the child is larger than a hash, returning the collapsed node as well
-// as a replacement for the original node with the child hashes cached in.
-func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, error) {
-	var err error
-
-	switch n := original.(type) {
-	case shortNode:
-		// Hash the short node's child, caching the newly hashed subtree
-		cached := n
-		cached.Key = common.CopyBytes(cached.Key)
-
-		n.Key = compactEncode(n.Key)
-		if _, ok := n.Val.(valueNode); !ok {
-			if n.Val, cached.Val, err = h.hash(n.Val, db, false); err != nil {
-				return n, original, err
-			}
-		}
-		if n.Val == nil {
-			n.Val = valueNode(nil) // Ensure that nil children are encoded as empty strings.
-		}
-		return n, cached, nil
-
-	case fullNode:
-		// Hash the full node's children, caching the newly hashed subtrees
-		cached := fullNode{dirty: n.dirty}
-
-		for i := 0; i < 16; i++ {
-			if n.Children[i] != nil {
-				if n.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false); err != nil {
-					return n, original, err
-				}
-			} else {
-				n.Children[i] = valueNode(nil) // Ensure that nil children are encoded as empty strings.
-			}
-		}
-		cached.Children[16] = n.Children[16]
-		if n.Children[16] == nil {
-			n.Children[16] = valueNode(nil)
-		}
-		return n, cached, nil
-
-	default:
-		// Value and hash nodes don't have children so they're left as were
-		return n, original, nil
-	}
-}
-
-func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
-	// Don't store hashes or empty nodes.
-	if _, isHash := n.(hashNode); n == nil || isHash {
-		return n, nil
-	}
-	// Generate the RLP encoding of the node
-	h.tmp.Reset()
-	if err := rlp.Encode(h.tmp, n); err != nil {
-		panic("encode error: " + err.Error())
-	}
-	if h.tmp.Len() < 32 && !force {
-		return n, nil // Nodes smaller than 32 bytes are stored inside their parent
-	}
-	// Larger nodes are replaced by their hash and stored in the database.
-	hash, _ := n.cache()
-	if hash == nil {
-		h.sha.Reset()
-		h.sha.Write(h.tmp.Bytes())
-		hash = hashNode(h.sha.Sum(nil))
-	}
-	if db != nil {
-		return hash, db.Put(hash, h.tmp.Bytes())
-	}
-	return hash, nil
 }

trie/trie_test.go

@@ -21,8 +21,11 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"os"
+	"reflect"
 	"testing"
+	"testing/quick"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
@@ -76,8 +79,6 @@ func TestMissingNode(t *testing.T) {
 	updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
 	root, _ := trie.Commit()
 
-	ClearGlobalCache()
-
 	trie, _ = New(root, db)
 	_, err := trie.TryGet([]byte("120000"))
 	if err != nil {
@@ -109,7 +110,6 @@ func TestMissingNode(t *testing.T) {
 	}
 
 	db.Delete(common.FromHex("e1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9"))
-	ClearGlobalCache()
 
 	trie, _ = New(root, db)
 	_, err = trie.TryGet([]byte("120000"))
@@ -300,41 +300,6 @@ func TestReplication(t *testing.T) {
 	}
 }
 
-func paranoiaCheck(t1 *Trie) (bool, *Trie) {
-	t2 := new(Trie)
-	it := NewIterator(t1)
-	for it.Next() {
-		t2.Update(it.Key, it.Value)
-	}
-	return t2.Hash() == t1.Hash(), t2
-}
-
-func TestParanoia(t *testing.T) {
-	t.Skip()
-	trie := newEmpty()
-
-	vals := []struct{ k, v string }{
-		{"do", "verb"},
-		{"ether", "wookiedoo"},
-		{"horse", "stallion"},
-		{"shaman", "horse"},
-		{"doge", "coin"},
-		{"ether", ""},
-		{"dog", "puppy"},
-		{"shaman", ""},
-		{"somethingveryoddindeedthis is", "myothernodedata"},
-	}
-	for _, val := range vals {
-		updateString(trie, val.k, val.v)
-	}
-	trie.Commit()
-
-	ok, t2 := paranoiaCheck(trie)
-	if !ok {
-		t.Errorf("trie paranoia check failed %x %x", trie.Hash(), t2.Hash())
-	}
-}
-
 // Not an actual test
 func TestOutput(t *testing.T) {
 	t.Skip()
@@ -359,44 +324,127 @@ func TestLargeValue(t *testing.T) {
 	trie.Update([]byte("key1"), []byte{99, 99, 99, 99})
 	trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32))
 	trie.Hash()
-
 }
 
-type kv struct {
+type randTestStep struct {
-	k, v []byte
+	op    int
-	t    bool
+	key   []byte // for opUpdate, opDelete, opGet
+	value []byte // for opUpdate
 }
 
-func TestLargeData(t *testing.T) {
+type randTest []randTestStep
-	trie := newEmpty()
-	vals := make(map[string]*kv)
 
-	for i := byte(0); i < 255; i++ {
+const (
-		value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
+	opUpdate = iota
-		value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false}
+	opDelete
-		trie.Update(value.k, value.v)
+	opGet
-		trie.Update(value2.k, value2.v)
+	opCommit
-		vals[string(value.k)] = value
+	opHash
-		vals[string(value2.k)] = value2
+	opReset
+	opItercheckhash
+	opMax // boundary value, not an actual op
+)
+
+func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
+	var allKeys [][]byte
+	genKey := func() []byte {
+		if len(allKeys) < 2 || r.Intn(100) < 10 {
+			// new key
+			key := make([]byte, r.Intn(50))
+			randRead(r, key)
+			allKeys = append(allKeys, key)
+			return key
+		}
+		// use existing key
+		return allKeys[r.Intn(len(allKeys))]
 	}
 
-	it := NewIterator(trie)
+	var steps randTest
-	for it.Next() {
+	for i := 0; i < size; i++ {
-		vals[string(it.Key)].t = true
+		step := randTestStep{op: r.Intn(opMax)}
+		switch step.op {
+		case opUpdate:
+			step.key = genKey()
+			step.value = make([]byte, 8)
+			binary.BigEndian.PutUint64(step.value, uint64(i))
+		case opGet, opDelete:
+			step.key = genKey()
+		}
+		steps = append(steps, step)
 	}
+	return reflect.ValueOf(steps)
+}
 
-	var untouched []*kv
+// rand.Rand provides a Read method in Go 1.7 and later, but
-	for _, value := range vals {
+// we can't use it yet.
-		if !value.t {
+func randRead(r *rand.Rand, b []byte) {
-			untouched = append(untouched, value)
+	pos := 0
+	val := 0
+	for n := 0; n < len(b); n++ {
+		if pos == 0 {
+			val = r.Int()
+			pos = 7
+		}
+		b[n] = byte(val)
+		val >>= 8
+		pos--
+	}
+}
+
+func runRandTest(rt randTest) bool {
+	db, _ := ethdb.NewMemDatabase()
+	tr, _ := New(common.Hash{}, db)
+	values := make(map[string]string) // tracks content of the trie
+
+	for _, step := range rt {
+		switch step.op {
+		case opUpdate:
+			tr.Update(step.key, step.value)
+			values[string(step.key)] = string(step.value)
+		case opDelete:
+			tr.Delete(step.key)
+			delete(values, string(step.key))
+		case opGet:
+			v := tr.Get(step.key)
+			want := values[string(step.key)]
+			if string(v) != want {
+				fmt.Printf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want)
+				return false
+			}
+		case opCommit:
+			if _, err := tr.Commit(); err != nil {
+				panic(err)
+			}
+		case opHash:
+			tr.Hash()
+		case opReset:
+			hash, err := tr.Commit()
+			if err != nil {
+				panic(err)
+			}
+			newtr, err := New(hash, db)
+			if err != nil {
+				panic(err)
+			}
+			tr = newtr
+		case opItercheckhash:
+			checktr, _ := New(common.Hash{}, nil)
+			it := tr.Iterator()
+			for it.Next() {
+				checktr.Update(it.Key, it.Value)
+			}
+			if tr.Hash() != checktr.Hash() {
+				fmt.Println("hashes not equal")
+				return false
+			}
 		}
 	}
+	return true
+}
 
-	if len(untouched) > 0 {
+func TestRandom(t *testing.T) {
-		t.Errorf("Missed %d nodes", len(untouched))
+	if err := quick.Check(runRandTest, nil); err != nil {
-		for _, value := range untouched {
+		t.Fatal(err)
-			t.Error(value)
-		}
 	}
 }