all: make indexer configurable (#17188)
This commit is contained in:
committed by
Péter Szilágyi
parent
c64d72bea2
commit
b69476b372
@ -48,6 +48,7 @@ var (
|
||||
// interface. It only does header validation during chain insertion.
|
||||
type LightChain struct {
|
||||
hc *core.HeaderChain
|
||||
indexerConfig *IndexerConfig
|
||||
chainDb ethdb.Database
|
||||
odr OdrBackend
|
||||
chainFeed event.Feed
|
||||
@ -81,13 +82,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.
|
||||
blockCache, _ := lru.New(blockCacheLimit)
|
||||
|
||||
bc := &LightChain{
|
||||
chainDb: odr.Database(),
|
||||
odr: odr,
|
||||
quit: make(chan struct{}),
|
||||
bodyCache: bodyCache,
|
||||
bodyRLPCache: bodyRLPCache,
|
||||
blockCache: blockCache,
|
||||
engine: engine,
|
||||
chainDb: odr.Database(),
|
||||
indexerConfig: odr.IndexerConfig(),
|
||||
odr: odr,
|
||||
quit: make(chan struct{}),
|
||||
bodyCache: bodyCache,
|
||||
bodyRLPCache: bodyRLPCache,
|
||||
blockCache: blockCache,
|
||||
engine: engine,
|
||||
}
|
||||
var err error
|
||||
bc.hc, err = core.NewHeaderChain(odr.Database(), config, bc.engine, bc.getProcInterrupt)
|
||||
@ -128,7 +130,7 @@ func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) {
|
||||
if self.odr.BloomIndexer() != nil {
|
||||
self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead)
|
||||
}
|
||||
log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*CHTFrequencyClient-1, "hash", cp.SectionHead)
|
||||
log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*self.indexerConfig.ChtSize-1, "hash", cp.SectionHead)
|
||||
}
|
||||
|
||||
func (self *LightChain) getProcInterrupt() bool {
|
||||
@ -472,7 +474,7 @@ func (self *LightChain) SyncCht(ctx context.Context) bool {
|
||||
head := self.CurrentHeader().Number.Uint64()
|
||||
sections, _, _ := self.odr.ChtIndexer().Sections()
|
||||
|
||||
latest := sections*CHTFrequencyClient - 1
|
||||
latest := sections*self.indexerConfig.ChtSize - 1
|
||||
if clique := self.hc.Config().Clique; clique != nil {
|
||||
latest -= latest % clique.Epoch // epoch snapshot for clique
|
||||
}
|
||||
|
@ -55,7 +55,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) {
|
||||
db := ethdb.NewMemDatabase()
|
||||
gspec := core.Genesis{Config: params.TestChainConfig}
|
||||
genesis := gspec.MustCommit(db)
|
||||
blockchain, _ := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFaker())
|
||||
blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker())
|
||||
|
||||
// Create and inject the requested chain
|
||||
if n == 0 {
|
||||
@ -265,7 +265,8 @@ func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.
|
||||
|
||||
type dummyOdr struct {
|
||||
OdrBackend
|
||||
db ethdb.Database
|
||||
db ethdb.Database
|
||||
indexerConfig *IndexerConfig
|
||||
}
|
||||
|
||||
func (odr *dummyOdr) Database() ethdb.Database {
|
||||
@ -276,6 +277,10 @@ func (odr *dummyOdr) Retrieve(ctx context.Context, req OdrRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (odr *dummyOdr) IndexerConfig() *IndexerConfig {
|
||||
return odr.indexerConfig
|
||||
}
|
||||
|
||||
// Tests that reorganizing a long difficult chain after a short easy one
|
||||
// overwrites the canonical numbers and links in the database.
|
||||
func TestReorgLongHeaders(t *testing.T) {
|
||||
|
@ -44,6 +44,7 @@ type OdrBackend interface {
|
||||
BloomTrieIndexer() *core.ChainIndexer
|
||||
BloomIndexer() *core.ChainIndexer
|
||||
Retrieve(ctx context.Context, req OdrRequest) error
|
||||
IndexerConfig() *IndexerConfig
|
||||
}
|
||||
|
||||
// OdrRequest is an interface for retrieval requests
|
||||
@ -136,6 +137,7 @@ func (req *ReceiptsRequest) StoreResult(db ethdb.Database) {
|
||||
// ChtRequest is the ODR request type for state/storage trie entries
|
||||
type ChtRequest struct {
|
||||
OdrRequest
|
||||
Config *IndexerConfig
|
||||
ChtNum, BlockNum uint64
|
||||
ChtRoot common.Hash
|
||||
Header *types.Header
|
||||
@ -155,6 +157,7 @@ func (req *ChtRequest) StoreResult(db ethdb.Database) {
|
||||
// BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure
|
||||
type BloomRequest struct {
|
||||
OdrRequest
|
||||
Config *IndexerConfig
|
||||
BloomTrieNum uint64
|
||||
BitIdx uint
|
||||
SectionIdxList []uint64
|
||||
@ -166,7 +169,7 @@ type BloomRequest struct {
|
||||
// StoreResult stores the retrieved data in local database
|
||||
func (req *BloomRequest) StoreResult(db ethdb.Database) {
|
||||
for i, sectionIdx := range req.SectionIdxList {
|
||||
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1)
|
||||
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*req.Config.BloomTrieSize-1)
|
||||
// if we don't have the canonical hash stored for this section head number, we'll still store it under
|
||||
// a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical
|
||||
// hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the
|
||||
|
@ -55,8 +55,9 @@ var (
|
||||
|
||||
type testOdr struct {
|
||||
OdrBackend
|
||||
sdb, ldb ethdb.Database
|
||||
disable bool
|
||||
indexerConfig *IndexerConfig
|
||||
sdb, ldb ethdb.Database
|
||||
disable bool
|
||||
}
|
||||
|
||||
func (odr *testOdr) Database() ethdb.Database {
|
||||
@ -92,6 +93,10 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (odr *testOdr) IndexerConfig() *IndexerConfig {
|
||||
return odr.indexerConfig
|
||||
}
|
||||
|
||||
type odrTestFn func(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error)
|
||||
|
||||
func TestOdrGetBlockLes1(t *testing.T) { testChainOdr(t, 1, odrGetBlock) }
|
||||
@ -258,7 +263,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
odr := &testOdr{sdb: sdb, ldb: ldb}
|
||||
odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig}
|
||||
lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
@ -53,16 +53,16 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ
|
||||
for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
|
||||
chtCount--
|
||||
if chtCount > 0 {
|
||||
sectionHeadNum = chtCount*CHTFrequencyClient - 1
|
||||
sectionHeadNum = chtCount*odr.IndexerConfig().ChtSize - 1
|
||||
sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1)
|
||||
canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum)
|
||||
}
|
||||
}
|
||||
}
|
||||
if number >= chtCount*CHTFrequencyClient {
|
||||
if number >= chtCount*odr.IndexerConfig().ChtSize {
|
||||
return nil, ErrNoTrustedCht
|
||||
}
|
||||
r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number}
|
||||
r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number, Config: odr.IndexerConfig()}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -175,9 +175,9 @@ func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number
|
||||
|
||||
// GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes
|
||||
func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxList []uint64) ([][]byte, error) {
|
||||
db := odr.Database()
|
||||
result := make([][]byte, len(sectionIdxList))
|
||||
var (
|
||||
db = odr.Database()
|
||||
result = make([][]byte, len(sectionIdxList))
|
||||
reqList []uint64
|
||||
reqIdx []int
|
||||
)
|
||||
@ -193,7 +193,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi
|
||||
for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
|
||||
bloomTrieCount--
|
||||
if bloomTrieCount > 0 {
|
||||
sectionHeadNum = bloomTrieCount*BloomTrieFrequency - 1
|
||||
sectionHeadNum = bloomTrieCount*odr.IndexerConfig().BloomTrieSize - 1
|
||||
sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1)
|
||||
canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum)
|
||||
}
|
||||
@ -201,7 +201,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi
|
||||
}
|
||||
|
||||
for i, sectionIdx := range sectionIdxList {
|
||||
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1)
|
||||
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*odr.IndexerConfig().BloomSize-1)
|
||||
// if we don't have the canonical hash stored for this section head number, we'll still look for
|
||||
// an entry with a zero sectionHead (we store it with zero section head too if we don't know it
|
||||
// at the time of the retrieval)
|
||||
@ -209,6 +209,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi
|
||||
if err == nil {
|
||||
result[i] = bloomBits
|
||||
} else {
|
||||
// TODO(rjl493456442) Convert sectionIndex to BloomTrie relative index
|
||||
if sectionIdx >= bloomTrieCount {
|
||||
return nil, ErrNoTrustedBloomTrie
|
||||
}
|
||||
@ -220,7 +221,8 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi
|
||||
return result, nil
|
||||
}
|
||||
|
||||
r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1, BitIdx: bitIdx, SectionIdxList: reqList}
|
||||
r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1,
|
||||
BitIdx: bitIdx, SectionIdxList: reqList, Config: odr.IndexerConfig()}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
|
@ -36,20 +36,75 @@ import (
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
)
|
||||
|
||||
const (
|
||||
// CHTFrequencyClient is the block frequency for creating CHTs on the client side.
|
||||
CHTFrequencyClient = 32768
|
||||
// IndexerConfig includes a set of configs for chain indexers.
|
||||
type IndexerConfig struct {
|
||||
// The block frequency for creating CHTs.
|
||||
ChtSize uint64
|
||||
|
||||
// CHTFrequencyServer is the block frequency for creating CHTs on the server side.
|
||||
// Eventually this can be merged back with the client version, but that requires a
|
||||
// full database upgrade, so that should be left for a suitable moment.
|
||||
CHTFrequencyServer = 4096
|
||||
// A special auxiliary field represents client's chtsize for server config, otherwise represents server's chtsize.
|
||||
PairChtSize uint64
|
||||
|
||||
HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available
|
||||
HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated
|
||||
// The number of confirmations needed to generate/accept a canonical hash help trie.
|
||||
ChtConfirms uint64
|
||||
|
||||
// The block frequency for creating new bloom bits.
|
||||
BloomSize uint64
|
||||
|
||||
// The number of confirmation needed before a bloom section is considered probably final and its rotated bits
|
||||
// are calculated.
|
||||
BloomConfirms uint64
|
||||
|
||||
// The block frequency for creating BloomTrie.
|
||||
BloomTrieSize uint64
|
||||
|
||||
// The number of confirmations needed to generate/accept a bloom trie.
|
||||
BloomTrieConfirms uint64
|
||||
}
|
||||
|
||||
var (
|
||||
// DefaultServerIndexerConfig wraps a set of configs as a default indexer config for server side.
|
||||
DefaultServerIndexerConfig = &IndexerConfig{
|
||||
ChtSize: params.CHTFrequencyServer,
|
||||
PairChtSize: params.CHTFrequencyClient,
|
||||
ChtConfirms: params.HelperTrieProcessConfirmations,
|
||||
BloomSize: params.BloomBitsBlocks,
|
||||
BloomConfirms: params.BloomConfirms,
|
||||
BloomTrieSize: params.BloomTrieFrequency,
|
||||
BloomTrieConfirms: params.HelperTrieProcessConfirmations,
|
||||
}
|
||||
// DefaultClientIndexerConfig wraps a set of configs as a default indexer config for client side.
|
||||
DefaultClientIndexerConfig = &IndexerConfig{
|
||||
ChtSize: params.CHTFrequencyClient,
|
||||
PairChtSize: params.CHTFrequencyServer,
|
||||
ChtConfirms: params.HelperTrieConfirmations,
|
||||
BloomSize: params.BloomBitsBlocksClient,
|
||||
BloomConfirms: params.HelperTrieConfirmations,
|
||||
BloomTrieSize: params.BloomTrieFrequency,
|
||||
BloomTrieConfirms: params.HelperTrieConfirmations,
|
||||
}
|
||||
// TestServerIndexerConfig wraps a set of configs as a test indexer config for server side.
|
||||
TestServerIndexerConfig = &IndexerConfig{
|
||||
ChtSize: 256,
|
||||
PairChtSize: 2048,
|
||||
ChtConfirms: 16,
|
||||
BloomSize: 256,
|
||||
BloomConfirms: 16,
|
||||
BloomTrieSize: 2048,
|
||||
BloomTrieConfirms: 16,
|
||||
}
|
||||
// TestClientIndexerConfig wraps a set of configs as a test indexer config for client side.
|
||||
TestClientIndexerConfig = &IndexerConfig{
|
||||
ChtSize: 2048,
|
||||
PairChtSize: 256,
|
||||
ChtConfirms: 128,
|
||||
BloomSize: 2048,
|
||||
BloomConfirms: 128,
|
||||
BloomTrieSize: 2048,
|
||||
BloomTrieConfirms: 128,
|
||||
}
|
||||
)
|
||||
|
||||
// TrustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with
|
||||
// trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with
|
||||
// the appropriate section index and head hash. It is used to start light syncing from this checkpoint
|
||||
// and avoid downloading the entire header chain while still being able to securely access old headers/logs.
|
||||
type TrustedCheckpoint struct {
|
||||
@ -84,9 +139,9 @@ var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{
|
||||
}
|
||||
|
||||
var (
|
||||
ErrNoTrustedCht = errors.New("No trusted canonical hash trie")
|
||||
ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie")
|
||||
ErrNoHeader = errors.New("Header not found")
|
||||
ErrNoTrustedCht = errors.New("no trusted canonical hash trie")
|
||||
ErrNoTrustedBloomTrie = errors.New("no trusted bloom trie")
|
||||
ErrNoHeader = errors.New("header not found")
|
||||
chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash
|
||||
ChtTablePrefix = "cht-"
|
||||
)
|
||||
@ -97,8 +152,8 @@ type ChtNode struct {
|
||||
Td *big.Int
|
||||
}
|
||||
|
||||
// GetChtRoot reads the CHT root assoctiated to the given section from the database
|
||||
// Note that sectionIdx is specified according to LES/1 CHT section size
|
||||
// GetChtRoot reads the CHT root associated to the given section from the database
|
||||
// Note that sectionIdx is specified according to LES/1 CHT section size.
|
||||
func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash {
|
||||
var encNumber [8]byte
|
||||
binary.BigEndian.PutUint64(encNumber[:], sectionIdx)
|
||||
@ -106,21 +161,15 @@ func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) c
|
||||
return common.BytesToHash(data)
|
||||
}
|
||||
|
||||
// GetChtV2Root reads the CHT root assoctiated to the given section from the database
|
||||
// Note that sectionIdx is specified according to LES/2 CHT section size
|
||||
func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash {
|
||||
return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead)
|
||||
}
|
||||
|
||||
// StoreChtRoot writes the CHT root assoctiated to the given section into the database
|
||||
// Note that sectionIdx is specified according to LES/1 CHT section size
|
||||
// StoreChtRoot writes the CHT root associated to the given section into the database
|
||||
// Note that sectionIdx is specified according to LES/1 CHT section size.
|
||||
func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) {
|
||||
var encNumber [8]byte
|
||||
binary.BigEndian.PutUint64(encNumber[:], sectionIdx)
|
||||
db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes())
|
||||
}
|
||||
|
||||
// ChtIndexerBackend implements core.ChainIndexerBackend
|
||||
// ChtIndexerBackend implements core.ChainIndexerBackend.
|
||||
type ChtIndexerBackend struct {
|
||||
diskdb, trieTable ethdb.Database
|
||||
odr OdrBackend
|
||||
@ -130,33 +179,24 @@ type ChtIndexerBackend struct {
|
||||
trie *trie.Trie
|
||||
}
|
||||
|
||||
// NewBloomTrieIndexer creates a BloomTrie chain indexer
|
||||
func NewChtIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer {
|
||||
var sectionSize, confirmReq uint64
|
||||
if clientMode {
|
||||
sectionSize = CHTFrequencyClient
|
||||
confirmReq = HelperTrieConfirmations
|
||||
} else {
|
||||
sectionSize = CHTFrequencyServer
|
||||
confirmReq = HelperTrieProcessConfirmations
|
||||
}
|
||||
idb := ethdb.NewTable(db, "chtIndex-")
|
||||
// NewChtIndexer creates a Cht chain indexer
|
||||
func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *core.ChainIndexer {
|
||||
trieTable := ethdb.NewTable(db, ChtTablePrefix)
|
||||
backend := &ChtIndexerBackend{
|
||||
diskdb: db,
|
||||
odr: odr,
|
||||
trieTable: trieTable,
|
||||
triedb: trie.NewDatabase(trieTable),
|
||||
sectionSize: sectionSize,
|
||||
sectionSize: size,
|
||||
}
|
||||
return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht")
|
||||
return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht")
|
||||
}
|
||||
|
||||
// fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the
|
||||
// ODR backend in order to be able to add new entries and calculate subsequent root hashes
|
||||
func (c *ChtIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error {
|
||||
batch := c.trieTable.NewBatch()
|
||||
r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1}
|
||||
r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1, Config: c.odr.IndexerConfig()}
|
||||
for {
|
||||
err := c.odr.Retrieve(ctx, r)
|
||||
switch err {
|
||||
@ -221,18 +261,13 @@ func (c *ChtIndexerBackend) Commit() error {
|
||||
}
|
||||
c.triedb.Commit(root, false)
|
||||
|
||||
if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 {
|
||||
log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root))
|
||||
if ((c.section+1)*c.sectionSize)%params.CHTFrequencyClient == 0 {
|
||||
log.Info("Storing CHT", "section", c.section*c.sectionSize/params.CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root))
|
||||
}
|
||||
StoreChtRoot(c.diskdb, c.section, c.lastHash, root)
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
BloomTrieFrequency = 32768
|
||||
ethBloomBitsSection = 4096
|
||||
)
|
||||
|
||||
var (
|
||||
bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash
|
||||
BloomTrieTablePrefix = "blt-"
|
||||
@ -255,33 +290,31 @@ func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root
|
||||
|
||||
// BloomTrieIndexerBackend implements core.ChainIndexerBackend
|
||||
type BloomTrieIndexerBackend struct {
|
||||
diskdb, trieTable ethdb.Database
|
||||
odr OdrBackend
|
||||
triedb *trie.Database
|
||||
section, parentSectionSize, bloomTrieRatio uint64
|
||||
trie *trie.Trie
|
||||
sectionHeads []common.Hash
|
||||
diskdb, trieTable ethdb.Database
|
||||
triedb *trie.Database
|
||||
odr OdrBackend
|
||||
section uint64
|
||||
parentSize uint64
|
||||
size uint64
|
||||
bloomTrieRatio uint64
|
||||
trie *trie.Trie
|
||||
sectionHeads []common.Hash
|
||||
}
|
||||
|
||||
// NewBloomTrieIndexer creates a BloomTrie chain indexer
|
||||
func NewBloomTrieIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer {
|
||||
func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uint64) *core.ChainIndexer {
|
||||
trieTable := ethdb.NewTable(db, BloomTrieTablePrefix)
|
||||
backend := &BloomTrieIndexerBackend{
|
||||
diskdb: db,
|
||||
odr: odr,
|
||||
trieTable: trieTable,
|
||||
triedb: trie.NewDatabase(trieTable),
|
||||
diskdb: db,
|
||||
odr: odr,
|
||||
trieTable: trieTable,
|
||||
triedb: trie.NewDatabase(trieTable),
|
||||
parentSize: parentSize,
|
||||
size: size,
|
||||
}
|
||||
idb := ethdb.NewTable(db, "bltIndex-")
|
||||
|
||||
if clientMode {
|
||||
backend.parentSectionSize = BloomTrieFrequency
|
||||
} else {
|
||||
backend.parentSectionSize = ethBloomBitsSection
|
||||
}
|
||||
backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize
|
||||
backend.bloomTrieRatio = size / parentSize
|
||||
backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio)
|
||||
return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, 0, time.Millisecond*100, "bloomtrie")
|
||||
return core.NewChainIndexer(db, ethdb.NewTable(db, "bltIndex-"), backend, size, 0, time.Millisecond*100, "bloomtrie")
|
||||
}
|
||||
|
||||
// fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the
|
||||
@ -296,7 +329,7 @@ func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section
|
||||
for i := 0; i < 20; i++ {
|
||||
go func() {
|
||||
for bitIndex := range indexCh {
|
||||
r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}}
|
||||
r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}, Config: b.odr.IndexerConfig()}
|
||||
for {
|
||||
if err := b.odr.Retrieve(ctx, r); err == ErrNoPeers {
|
||||
// if there are no peers to serve, retry later
|
||||
@ -351,9 +384,9 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las
|
||||
|
||||
// Process implements core.ChainIndexerBackend
|
||||
func (b *BloomTrieIndexerBackend) Process(ctx context.Context, header *types.Header) error {
|
||||
num := header.Number.Uint64() - b.section*BloomTrieFrequency
|
||||
if (num+1)%b.parentSectionSize == 0 {
|
||||
b.sectionHeads[num/b.parentSectionSize] = header.Hash()
|
||||
num := header.Number.Uint64() - b.section*b.size
|
||||
if (num+1)%b.parentSize == 0 {
|
||||
b.sectionHeads[num/b.parentSize] = header.Hash()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -372,7 +405,7 @@ func (b *BloomTrieIndexerBackend) Commit() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8))
|
||||
decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSize/8))
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
@ -397,6 +430,5 @@ func (b *BloomTrieIndexerBackend) Commit() error {
|
||||
sectionHead := b.sectionHeads[b.bloomTrieRatio-1]
|
||||
log.Info("Storing bloom trie", "section", b.section, "head", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression", float64(compSize)/float64(decompSize))
|
||||
StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -47,7 +47,7 @@ func TestNodeIterator(t *testing.T) {
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
odr := &testOdr{sdb: fulldb, ldb: lightdb}
|
||||
odr := &testOdr{sdb: fulldb, ldb: lightdb, indexerConfig: TestClientIndexerConfig}
|
||||
head := blockchain.CurrentHeader()
|
||||
lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root)
|
||||
fullTrie, _ := state.NewDatabase(fulldb).OpenTrie(head.Root)
|
||||
|
@ -94,7 +94,7 @@ func TestTxPool(t *testing.T) {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
odr := &testOdr{sdb: sdb, ldb: ldb}
|
||||
odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig}
|
||||
relay := &testTxRelay{
|
||||
send: make(chan int, 1),
|
||||
discard: make(chan int, 1),
|
||||
|
Reference in New Issue
Block a user