core/state, cmd/geth: streaming json output for dump command (#15475)

* core/state, cmd/geth: streaming json output dump cmd + optional code+storage

* dump: add option to continue even if preimages are missing

* core, evm: lint nits

* cmd: use local flags for dump, omit empty code/storage

* core/state: fix state dump test

core/state/dump.go

@@ -21,61 +21,133 @@ import (
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
+// DumpAccount represents an account in the state
 type DumpAccount struct {
-	Balance  string            `json:"balance"`
-	Nonce    uint64            `json:"nonce"`
-	Root     string            `json:"root"`
-	CodeHash string            `json:"codeHash"`
-	Code     string            `json:"code"`
-	Storage  map[string]string `json:"storage"`
+	Balance   string                 `json:"balance"`
+	Nonce     uint64                 `json:"nonce"`
+	Root      string                 `json:"root"`
+	CodeHash  string                 `json:"codeHash"`
+	Code      string                 `json:"code,omitempty"`
+	Storage   map[common.Hash]string `json:"storage,omitempty"`
+	Address   *common.Address        `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
+	SecureKey hexutil.Bytes          `json:"key,omitempty"`     // If we don't have address, we can output the key
+
 }
 
+// Dump represents the full dump in a collected format, as one large map
 type Dump struct {
-	Root     string                 `json:"root"`
-	Accounts map[string]DumpAccount `json:"accounts"`
+	Root     string                         `json:"root"`
+	Accounts map[common.Address]DumpAccount `json:"accounts"`
 }
 
-func (self *StateDB) RawDump() Dump {
-	dump := Dump{
-		Root:     fmt.Sprintf("%x", self.trie.Hash()),
-		Accounts: make(map[string]DumpAccount),
-	}
+// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
+type iterativeDump json.Encoder
 
+// Collector interface which the state trie calls during iteration
+type collector interface {
+	onRoot(common.Hash)
+	onAccount(common.Address, DumpAccount)
+}
+
+func (self *Dump) onRoot(root common.Hash) {
+	self.Root = fmt.Sprintf("%x", root)
+}
+
+func (self *Dump) onAccount(addr common.Address, account DumpAccount) {
+	self.Accounts[addr] = account
+}
+
+func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) {
+	dumpAccount := &DumpAccount{
+		Balance:   account.Balance,
+		Nonce:     account.Nonce,
+		Root:      account.Root,
+		CodeHash:  account.CodeHash,
+		Code:      account.Code,
+		Storage:   account.Storage,
+		SecureKey: account.SecureKey,
+		Address:   nil,
+	}
+	if addr != (common.Address{}) {
+		dumpAccount.Address = &addr
+	}
+	(*json.Encoder)(&self).Encode(dumpAccount)
+}
+func (self iterativeDump) onRoot(root common.Hash) {
+	(*json.Encoder)(&self).Encode(struct {
+		Root common.Hash `json:"root"`
+	}{root})
+}
+
+func (self *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
+	emptyAddress := (common.Address{})
+	missingPreimages := 0
+	c.onRoot(self.trie.Hash())
 	it := trie.NewIterator(self.trie.NodeIterator(nil))
 	for it.Next() {
-		addr := self.trie.GetKey(it.Key)
 		var data Account
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
 			panic(err)
 		}
-
-		obj := newObject(nil, common.BytesToAddress(addr), data)
+		addr := common.BytesToAddress(self.trie.GetKey(it.Key))
+		obj := newObject(nil, addr, data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,
 			Root:     common.Bytes2Hex(data.Root[:]),
 			CodeHash: common.Bytes2Hex(data.CodeHash),
-			Code:     common.Bytes2Hex(obj.Code(self.db)),
-			Storage:  make(map[string]string),
 		}
-		storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
-		for storageIt.Next() {
-			account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
+		if emptyAddress == addr {
+			// Preimage missing
+			missingPreimages++
+			if excludeMissingPreimages {
+				continue
+			}
+			account.SecureKey = it.Key
 		}
-		dump.Accounts[common.Bytes2Hex(addr)] = account
+		if !excludeCode {
+			account.Code = common.Bytes2Hex(obj.Code(self.db))
+		}
+		if !excludeStorage {
+			account.Storage = make(map[common.Hash]string)
+			storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
+			for storageIt.Next() {
+				account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
+			}
+		}
+		c.onAccount(addr, account)
+	}
+	if missingPreimages > 0 {
+		log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
 	}
-	return dump
 }
 
-func (self *StateDB) Dump() []byte {
-	json, err := json.MarshalIndent(self.RawDump(), "", "    ")
+// RawDump returns the entire state an a single large object
+func (self *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
+	dump := &Dump{
+		Accounts: make(map[common.Address]DumpAccount),
+	}
+	self.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
+	return *dump
+}
+
+// Dump returns a JSON string representing the entire state as a single json-object
+func (self *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
+	dump := self.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
+	json, err := json.MarshalIndent(dump, "", "    ")
 	if err != nil {
 		fmt.Println("dump err", err)
 	}
-
 	return json
 }
+
+// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
+func (self *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
+	self.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages)
+}