cmd, core, eth: background transaction indexing (#20302)
* cmd, core, eth: init tx lookup in background * core/rawdb: tiny log fixes to make it clearer what's happening * core, eth: fix rebase errors * core/rawdb: make reindexing less generic, but more optimal * rlp: implement rlp list iterator * core/rawdb: new implementation of tx indexing/unindex using generic tx iterator and hashing rlp-data * core/rawdb, cmd/utils: fix review concerns * cmd/utils: fix merge issue * core/rawdb: add some log formatting polishes Co-authored-by: rjl493456442 <garyrong0905@gmail.com> Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
committed by
GitHub
parent
6f54ae24cd
commit
4535230059
@ -190,7 +190,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
|
||||
SnapshotLimit: config.SnapshotCache,
|
||||
}
|
||||
)
|
||||
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve)
|
||||
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -105,6 +105,8 @@ type Config struct {
|
||||
NoPruning bool // Whether to disable pruning and flush everything to disk
|
||||
NoPrefetch bool // Whether to disable prefetching and only load state on demand
|
||||
|
||||
TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
|
||||
|
||||
// Whitelist of required block number -> hash values to accept
|
||||
Whitelist map[uint64]common.Hash `toml:"-"`
|
||||
|
||||
|
@ -24,6 +24,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
|
||||
DiscoveryURLs []string
|
||||
NoPruning bool
|
||||
NoPrefetch bool
|
||||
TxLookupLimit uint64 `toml:",omitempty"`
|
||||
Whitelist map[uint64]common.Hash `toml:"-"`
|
||||
LightServ int `toml:",omitempty"`
|
||||
LightIngress int `toml:",omitempty"`
|
||||
@ -60,6 +61,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
|
||||
enc.DiscoveryURLs = c.DiscoveryURLs
|
||||
enc.NoPruning = c.NoPruning
|
||||
enc.NoPrefetch = c.NoPrefetch
|
||||
enc.TxLookupLimit = c.TxLookupLimit
|
||||
enc.Whitelist = c.Whitelist
|
||||
enc.LightServ = c.LightServ
|
||||
enc.LightIngress = c.LightIngress
|
||||
@ -98,6 +100,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
|
||||
DiscoveryURLs []string
|
||||
NoPruning *bool
|
||||
NoPrefetch *bool
|
||||
TxLookupLimit *uint64 `toml:",omitempty"`
|
||||
Whitelist map[uint64]common.Hash `toml:"-"`
|
||||
LightServ *int `toml:",omitempty"`
|
||||
LightIngress *int `toml:",omitempty"`
|
||||
@ -149,6 +152,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
|
||||
if dec.NoPrefetch != nil {
|
||||
c.NoPrefetch = *dec.NoPrefetch
|
||||
}
|
||||
if dec.TxLookupLimit != nil {
|
||||
c.TxLookupLimit = *dec.TxLookupLimit
|
||||
}
|
||||
if dec.Whitelist != nil {
|
||||
c.Whitelist = dec.Whitelist
|
||||
}
|
||||
|
@ -72,6 +72,7 @@ type ProtocolManager struct {
|
||||
|
||||
txpool txPool
|
||||
blockchain *core.BlockChain
|
||||
chaindb ethdb.Database
|
||||
maxPeers int
|
||||
|
||||
downloader *downloader.Downloader
|
||||
@ -108,6 +109,7 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh
|
||||
eventMux: mux,
|
||||
txpool: txpool,
|
||||
blockchain: blockchain,
|
||||
chaindb: chaindb,
|
||||
peers: newPeerSet(),
|
||||
whitelist: whitelist,
|
||||
txsyncCh: make(chan *txsync),
|
||||
|
@ -491,7 +491,7 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo
|
||||
}
|
||||
}
|
||||
// Create a checkpoint aware protocol manager
|
||||
blockchain, err := core.NewBlockChain(db, nil, config, ethash.NewFaker(), vm.Config{}, nil)
|
||||
blockchain, err := core.NewBlockChain(db, nil, config, ethash.NewFaker(), vm.Config{}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create new blockchain: %v", err)
|
||||
}
|
||||
@ -578,7 +578,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
|
||||
gspec = &core.Genesis{Config: config}
|
||||
genesis = gspec.MustCommit(db)
|
||||
)
|
||||
blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil)
|
||||
blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create new blockchain: %v", err)
|
||||
}
|
||||
@ -639,7 +639,7 @@ func TestBroadcastMalformedBlock(t *testing.T) {
|
||||
gspec = &core.Genesis{Config: config}
|
||||
genesis = gspec.MustCommit(db)
|
||||
)
|
||||
blockchain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
|
||||
blockchain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create new blockchain: %v", err)
|
||||
}
|
||||
|
@ -62,7 +62,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func
|
||||
Alloc: core.GenesisAlloc{testBank: {Balance: big.NewInt(1000000)}},
|
||||
}
|
||||
genesis = gspec.MustCommit(db)
|
||||
blockchain, _ = core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil)
|
||||
blockchain, _ = core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil)
|
||||
)
|
||||
chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator)
|
||||
if _, err := blockchain.InsertChain(chain); err != nil {
|
||||
|
@ -175,8 +175,8 @@ func TestForkIDSplit(t *testing.T) {
|
||||
genesisNoFork = gspecNoFork.MustCommit(dbNoFork)
|
||||
genesisProFork = gspecProFork.MustCommit(dbProFork)
|
||||
|
||||
chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil)
|
||||
chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil)
|
||||
chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil)
|
||||
chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil)
|
||||
|
||||
blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil)
|
||||
blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil)
|
||||
|
19
eth/sync.go
19
eth/sync.go
@ -23,6 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/eth/downloader"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
@ -285,6 +286,24 @@ func (cs *chainSyncer) startSync(op *chainSyncOp) {
|
||||
|
||||
// doSync synchronizes the local blockchain with a remote peer.
|
||||
func (pm *ProtocolManager) doSync(op *chainSyncOp) error {
|
||||
if op.mode == downloader.FastSync {
|
||||
// Before launch the fast sync, we have to ensure user uses the same
|
||||
// txlookup limit.
|
||||
// The main concern here is: during the fast sync Geth won't index the
|
||||
// block(generate tx indices) before the HEAD-limit. But if user changes
|
||||
// the limit in the next fast sync(e.g. user kill Geth manually and
|
||||
// restart) then it will be hard for Geth to figure out the oldest block
|
||||
// has been indexed. So here for the user-experience wise, it's non-optimal
|
||||
// that user can't change limit during the fast sync. If changed, Geth
|
||||
// will just blindly use the original one.
|
||||
limit := pm.blockchain.TxLookupLimit()
|
||||
if stored := rawdb.ReadFastTxLookupLimit(pm.chaindb); stored == nil {
|
||||
rawdb.WriteFastTxLookupLimit(pm.chaindb, limit)
|
||||
} else if *stored != limit {
|
||||
pm.blockchain.SetTxLookupLimit(*stored)
|
||||
log.Warn("Update txLookup limit", "provided", limit, "updated", *stored)
|
||||
}
|
||||
}
|
||||
// Run the sync cycle, and disable fast sync if we're past the pivot block
|
||||
err := pm.downloader.Synchronise(op.peer.id, op.head, op.td, op.mode)
|
||||
if err != nil {
|
||||
|
Reference in New Issue
Block a user