gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

eth: add debug_accountRange API (#19645)

Browse Source 
This new API allows reading accounts and their content by address range.

Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Felix Lange <fjl@twurst.com>
This commit is contained in:
Wenbiao Zheng
2020-03-31 18:08:44 +08:00
committed by GitHub
parent c56f4fa808
commit 03fe9de2cb
3 changed files with 102 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
core/state/dump.go
View File

@@ -27,7 +27,7 @@ import (
"github.com/ethereum/go-ethereum/trie"
)
// DumpAccount represents an account in the state
// DumpAccount represents an account in the state.
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
@@ -40,17 +40,24 @@ type DumpAccount struct {
}
// Dump represents the full dump in a collected format, as one large map
// Dump represents the full dump in a collected format, as one large map.
type Dump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
}
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively.
type iterativeDump struct {
*json.Encoder
}
// IteratorDump is an implementation for iterating over data.
type IteratorDump struct {
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
Next []byte `json:"next,omitempty"` // nil if no more accounts
}
// Collector interface which the state trie calls during iteration
type collector interface {
onRoot(common.Hash)
@@ -64,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) {
func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d *IteratorDump) onRoot(root common.Hash) {
d.Root = fmt.Sprintf("%x", root)
}
func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) {
d.Accounts[addr] = account
}
func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
dumpAccount := &DumpAccount{
@@ -88,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) {
}{root})
}
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
emptyAddress := (common.Address{})
missingPreimages := 0
c.onRoot(s.trie.Hash())
it := trie.NewIterator(s.trie.NodeIterator(nil))
var count int
it := trie.NewIterator(s.trie.NodeIterator(start))
for it.Next() {
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
@@ -130,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP
}
}
c.onAccount(addr, account)
count++
if maxResults > 0 && count >= maxResults {
if it.Next() {
nextKey = it.Key
}
break
}
}
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
return nextKey
}
// RawDump returns the entire state an a single large object
@@ -141,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
return *dump
}
@@ -157,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages)
s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
}
// IteratorDump dumps out a batch of accounts starts with the given start key
func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
iterator := &IteratorDump{
Accounts: make(map[common.Address]DumpAccount),
}
iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
return *iterator
}

88
eth/api.go
View File

@@ -351,70 +351,50 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs,
return results, nil
}
// AccountRangeResult returns a mapping from the hash of an account addresses
// to its preimage. It will return the JSON null if no preimage is found.
// Since a query can return a limited amount of results, a "next" field is
// also present for paging.
type AccountRangeResult struct {
Accounts map[common.Hash]*common.Address `json:"accounts"`
Next common.Hash `json:"next"`
}
func accountRange(st state.Trie, start *common.Hash, maxResults int) (AccountRangeResult, error) {
if start == nil {
start = &common.Hash{0}
}
it := trie.NewIterator(st.NodeIterator(start.Bytes()))
result := AccountRangeResult{Accounts: make(map[common.Hash]*common.Address), Next: common.Hash{}}
if maxResults > AccountRangeMaxResults {
maxResults = AccountRangeMaxResults
}
for i := 0; i < maxResults && it.Next(); i++ {
if preimage := st.GetKey(it.Key); preimage != nil {
addr := &common.Address{}
addr.SetBytes(preimage)
result.Accounts[common.BytesToHash(it.Key)] = addr
} else {
result.Accounts[common.BytesToHash(it.Key)] = nil
}
}
if it.Next() {
result.Next = common.BytesToHash(it.Key)
}
return result, nil
}
// AccountRangeMaxResults is the maximum number of results to be returned per call
const AccountRangeMaxResults = 256
// AccountRange enumerates all accounts in the latest state
func (api *PrivateDebugAPI) AccountRange(ctx context.Context, start *common.Hash, maxResults int) (AccountRangeResult, error) {
var statedb *state.StateDB
// AccountRangeAt enumerates all accounts in the given block and start point in paging request
func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) {
var stateDb *state.StateDB
var err error
block := api.eth.blockchain.CurrentBlock()
if len(block.Transactions()) == 0 {
statedb, err = api.computeStateDB(block, defaultTraceReexec)
if err != nil {
return AccountRangeResult{}, err
if number, ok := blockNrOrHash.Number(); ok {
if number == rpc.PendingBlockNumber {
// If we're dumping the pending state, we need to request
// both the pending block as well as the pending state from
// the miner and operate on those
_, stateDb = api.eth.miner.Pending()
} else {
var block *types.Block
if number == rpc.LatestBlockNumber {
block = api.eth.blockchain.CurrentBlock()
} else {
block = api.eth.blockchain.GetBlockByNumber(uint64(number))
}
if block == nil {
return state.IteratorDump{}, fmt.Errorf("block #%d not found", number)
}
stateDb, err = api.eth.BlockChain().StateAt(block.Root())
if err != nil {
return state.IteratorDump{}, err
}
}
} else {
_, _, statedb, err = api.computeTxEnv(block.Hash(), len(block.Transactions())-1, 0)
} else if hash, ok := blockNrOrHash.Hash(); ok {
block := api.eth.blockchain.GetBlockByHash(hash)
if block == nil {
return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex())
}
stateDb, err = api.eth.BlockChain().StateAt(block.Root())
if err != nil {
return AccountRangeResult{}, err
return state.IteratorDump{}, err
}
}
trie, err := statedb.Database().OpenTrie(block.Header().Root)
if err != nil {
return AccountRangeResult{}, err
if maxResults > AccountRangeMaxResults || maxResults <= 0 {
maxResults = AccountRangeMaxResults
}
return accountRange(trie, start, maxResults)
return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil
}
// StorageRangeResult is the result of a debug_storageRangeAt API call.
@@ -431,7 +411,7 @@ type storageEntry struct {
}
// StorageRangeAt returns the storage at the given block height and transaction index.
func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
_, _, statedb, err := api.computeTxEnv(blockHash, txIndex, 0)
if err != nil {
return StorageRangeResult{}, err

98
eth/api_test.go
View File

@@ -33,29 +33,24 @@ import (
var dumper = spew.ConfigState{Indent: " "}
func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start *common.Hash, requestedNum int, expectedNum int) AccountRangeResult {
result, err := accountRange(*trie, start, requestedNum)
if err != nil {
t.Fatal(err)
}
func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump {
result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum)
if len(result.Accounts) != expectedNum {
t.Fatalf("expected %d results. Got %d", expectedNum, len(result.Accounts))
t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts))
}
for _, address := range result.Accounts {
if address == nil {
t.Fatalf("null address returned")
for address := range result.Accounts {
if address == (common.Address{}) {
t.Fatalf("empty address returned")
}
if !statedb.Exist(*address) {
if !statedb.Exist(address) {
t.Fatalf("account not found in state %s", address.Hex())
}
}
return result
}
type resultHash []*common.Hash
type resultHash []common.Hash
func (h resultHash) Len() int { return len(h) }
func (h resultHash) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
@@ -80,7 +75,6 @@ func TestAccountRange(t *testing.T) {
m[addr] = true
}
}
state.Commit(true)
root := state.IntermediateRoot(true)
@@ -88,68 +82,40 @@ func TestAccountRange(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Logf("test getting number of results less than max")
accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)
t.Logf("test getting number of results greater than max %d", AccountRangeMaxResults)
accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults*2, AccountRangeMaxResults)
t.Logf("test with empty 'start' hash")
accountRangeTest(t, &trie, state, nil, AccountRangeMaxResults, AccountRangeMaxResults)
t.Logf("test pagination")
accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)
// test pagination
firstResult := accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults, AccountRangeMaxResults)
t.Logf("test pagination 2")
secondResult := accountRangeTest(t, &trie, state, &firstResult.Next, AccountRangeMaxResults, AccountRangeMaxResults)
firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults)
secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults)
hList := make(resultHash, 0)
for h1, addr1 := range firstResult.Accounts {
h := &common.Hash{}
h.SetBytes(h1.Bytes())
hList = append(hList, h)
for h2, addr2 := range secondResult.Accounts {
// Make sure that the hashes aren't the same
if bytes.Equal(h1.Bytes(), h2.Bytes()) {
t.Fatalf("pagination test failed: results should not overlap")
}
// If either address is nil, then it makes no sense to compare
// them as they might be two different accounts.
if addr1 == nil || addr2 == nil {
continue
}
// Since the two hashes are different, they should not have
// the same preimage, but let's check anyway in case there
// is a bug in the (hash, addr) map generation code.
if bytes.Equal(addr1.Bytes(), addr2.Bytes()) {
t.Fatalf("pagination test failed: addresses should not repeat")
}
for addr1 := range firstResult.Accounts {
// If address is empty, then it makes no sense to compare
// them as they might be two different accounts.
if addr1 == (common.Address{}) {
continue
}
if _, duplicate := secondResult.Accounts[addr1]; duplicate {
t.Fatalf("pagination test failed: results should not overlap")
}
hList = append(hList, crypto.Keccak256Hash(addr1.Bytes()))
}
// Test to see if it's possible to recover from the middle of the previous
// set and get an even split between the first and second sets.
t.Logf("test random access pagination")
sort.Sort(hList)
middleH := hList[AccountRangeMaxResults/2]
middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults)
innone, infirst, insecond := 0, 0, 0
missing, infirst, insecond := 0, 0, 0
for h := range middleResult.Accounts {
if _, ok := firstResult.Accounts[h]; ok {
infirst++
} else if _, ok := secondResult.Accounts[h]; ok {
insecond++
} else {
innone++
missing++
}
}
if innone != 0 {
t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", innone)
if missing != 0 {
t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", missing)
}
if infirst != AccountRangeMaxResults/2 {
t.Fatalf("Imbalance in the number of first-test results: %d != %d", infirst, AccountRangeMaxResults/2)
@@ -164,20 +130,10 @@ func TestEmptyAccountRange(t *testing.T) {
statedb = state.NewDatabase(rawdb.NewMemoryDatabase())
state, _ = state.New(common.Hash{}, statedb, nil)
)
state.Commit(true)
root := state.IntermediateRoot(true)
trie, err := statedb.OpenTrie(root)
if err != nil {
t.Fatal(err)
}
results, err := accountRange(trie, &common.Hash{0x0}, AccountRangeMaxResults)
if err != nil {
t.Fatalf("Empty results should not trigger an error: %v", err)
}
if results.Next != common.HexToHash("0") {
state.IntermediateRoot(true)
results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults)
if bytes.Equal(results.Next, (common.Hash{}).Bytes()) {
t.Fatalf("Empty results should not return a second page")
}
if len(results.Accounts) != 0 {