cmd/geth, eth, core: snapshot dump + unify with trie dump (#22795)
* cmd/geth, eth, core: snapshot dump + unify with trie dump * cmd/evm: dump API fixes * cmd/geth, core, eth: fix some remaining errors * cmd/evm: dump - add limit, support address startkey, address review concerns * cmd, core/state, eth: minor polishes, fix snap dump crash, unify format Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
Martin Holst Swende
committed by
GitHub
GitHub
parent
1cca781a02
commit
addd8824cf
@ -19,6 +19,7 @@ package state
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
@ -27,6 +28,16 @@ import (
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
)
|
||||
|
||||
// DumpConfig is a set of options to control what portions of the statewill be
|
||||
// iterated and collected.
|
||||
type DumpConfig struct {
|
||||
SkipCode bool
|
||||
SkipStorage bool
|
||||
OnlyWithAddresses bool
|
||||
Start []byte
|
||||
Max uint64
|
||||
}
|
||||
|
||||
// DumpCollector interface which the state trie calls during iteration
|
||||
type DumpCollector interface {
|
||||
// OnRoot is called with the state root
|
||||
@ -39,9 +50,9 @@ type DumpCollector interface {
|
||||
type DumpAccount struct {
|
||||
Balance string `json:"balance"`
|
||||
Nonce uint64 `json:"nonce"`
|
||||
Root string `json:"root"`
|
||||
CodeHash string `json:"codeHash"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Root hexutil.Bytes `json:"root"`
|
||||
CodeHash hexutil.Bytes `json:"codeHash"`
|
||||
Code hexutil.Bytes `json:"code,omitempty"`
|
||||
Storage map[common.Hash]string `json:"storage,omitempty"`
|
||||
Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
|
||||
SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key
|
||||
@ -111,38 +122,50 @@ func (d iterativeDump) OnRoot(root common.Hash) {
|
||||
}{root})
|
||||
}
|
||||
|
||||
func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
|
||||
missingPreimages := 0
|
||||
// DumpToCollector iterates the state according to the given options and inserts
|
||||
// the items into a collector for aggregation or serialization.
|
||||
func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) {
|
||||
// Sanitize the input to allow nil configs
|
||||
if conf == nil {
|
||||
conf = new(DumpConfig)
|
||||
}
|
||||
var (
|
||||
missingPreimages int
|
||||
accounts uint64
|
||||
start = time.Now()
|
||||
logged = time.Now()
|
||||
)
|
||||
log.Info("Trie dumping started", "root", s.trie.Hash())
|
||||
c.OnRoot(s.trie.Hash())
|
||||
|
||||
var count int
|
||||
it := trie.NewIterator(s.trie.NodeIterator(start))
|
||||
it := trie.NewIterator(s.trie.NodeIterator(conf.Start))
|
||||
for it.Next() {
|
||||
var data Account
|
||||
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
account := DumpAccount{
|
||||
Balance: data.Balance.String(),
|
||||
Nonce: data.Nonce,
|
||||
Root: common.Bytes2Hex(data.Root[:]),
|
||||
CodeHash: common.Bytes2Hex(data.CodeHash),
|
||||
Balance: data.Balance.String(),
|
||||
Nonce: data.Nonce,
|
||||
Root: data.Root[:],
|
||||
CodeHash: data.CodeHash,
|
||||
SecureKey: it.Key,
|
||||
}
|
||||
addrBytes := s.trie.GetKey(it.Key)
|
||||
if addrBytes == nil {
|
||||
// Preimage missing
|
||||
missingPreimages++
|
||||
if excludeMissingPreimages {
|
||||
if conf.OnlyWithAddresses {
|
||||
continue
|
||||
}
|
||||
account.SecureKey = it.Key
|
||||
}
|
||||
addr := common.BytesToAddress(addrBytes)
|
||||
obj := newObject(s, addr, data)
|
||||
if !excludeCode {
|
||||
account.Code = common.Bytes2Hex(obj.Code(s.db))
|
||||
if !conf.SkipCode {
|
||||
account.Code = obj.Code(s.db)
|
||||
}
|
||||
if !excludeStorage {
|
||||
if !conf.SkipStorage {
|
||||
account.Storage = make(map[common.Hash]string)
|
||||
storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil))
|
||||
for storageIt.Next() {
|
||||
@ -155,8 +178,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage,
|
||||
}
|
||||
}
|
||||
c.OnAccount(addr, account)
|
||||
count++
|
||||
if maxResults > 0 && count >= maxResults {
|
||||
accounts++
|
||||
if time.Since(logged) > 8*time.Second {
|
||||
log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts,
|
||||
"elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
if conf.Max > 0 && accounts >= conf.Max {
|
||||
if it.Next() {
|
||||
nextKey = it.Key
|
||||
}
|
||||
@ -166,22 +194,24 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage,
|
||||
if missingPreimages > 0 {
|
||||
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
|
||||
}
|
||||
log.Info("Trie dumping complete", "accounts", accounts,
|
||||
"elapsed", common.PrettyDuration(time.Since(start)))
|
||||
|
||||
return nextKey
|
||||
}
|
||||
|
||||
// RawDump returns the entire state an a single large object
|
||||
func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
|
||||
func (s *StateDB) RawDump(opts *DumpConfig) Dump {
|
||||
dump := &Dump{
|
||||
Accounts: make(map[common.Address]DumpAccount),
|
||||
}
|
||||
s.DumpToCollector(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
|
||||
s.DumpToCollector(dump, opts)
|
||||
return *dump
|
||||
}
|
||||
|
||||
// Dump returns a JSON string representing the entire state as a single json-object
|
||||
func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
|
||||
dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
|
||||
func (s *StateDB) Dump(opts *DumpConfig) []byte {
|
||||
dump := s.RawDump(opts)
|
||||
json, err := json.MarshalIndent(dump, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println("Dump err", err)
|
||||
@ -190,15 +220,15 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
|
||||
}
|
||||
|
||||
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
|
||||
func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
|
||||
s.DumpToCollector(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
|
||||
func (s *StateDB) IterativeDump(opts *DumpConfig, output *json.Encoder) {
|
||||
s.DumpToCollector(iterativeDump{output}, opts)
|
||||
}
|
||||
|
||||
// IteratorDump dumps out a batch of accounts starts with the given start key
|
||||
func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
|
||||
func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump {
|
||||
iterator := &IteratorDump{
|
||||
Accounts: make(map[common.Address]DumpAccount),
|
||||
}
|
||||
iterator.Next = s.DumpToCollector(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
|
||||
iterator.Next = s.DumpToCollector(iterator, opts)
|
||||
return *iterator
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user