les: historical data garbage collection (#19570)
This change introduces garbage collection for the light client. Historical chain data is deleted periodically. If you want to disable the GC, use the --light.nopruning flag.
This commit is contained in:
gary rong
GitHub
@ -19,6 +19,7 @@ package light
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core"
|
||||
@ -30,65 +31,83 @@ import (
|
||||
|
||||
var sha3Nil = crypto.Keccak256Hash(nil)
|
||||
|
||||
// GetHeaderByNumber retrieves the canonical block header corresponding to the
|
||||
// given number.
|
||||
func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) {
|
||||
// Try to find it in the local database first.
|
||||
db := odr.Database()
|
||||
hash := rawdb.ReadCanonicalHash(db, number)
|
||||
if (hash != common.Hash{}) {
|
||||
// if there is a canonical hash, there is a header too
|
||||
header := rawdb.ReadHeader(db, hash, number)
|
||||
if header == nil {
|
||||
panic("Canonical hash present but header not found")
|
||||
}
|
||||
return header, nil
|
||||
}
|
||||
|
||||
var (
|
||||
chtCount, sectionHeadNum uint64
|
||||
sectionHead common.Hash
|
||||
)
|
||||
if odr.ChtIndexer() != nil {
|
||||
chtCount, sectionHeadNum, sectionHead = odr.ChtIndexer().Sections()
|
||||
canonicalHash := rawdb.ReadCanonicalHash(db, sectionHeadNum)
|
||||
// if the CHT was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too
|
||||
for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
|
||||
chtCount--
|
||||
if chtCount > 0 {
|
||||
sectionHeadNum = chtCount*odr.IndexerConfig().ChtSize - 1
|
||||
sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1)
|
||||
canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum)
|
||||
}
|
||||
// If there is a canonical hash, there should have a header too.
|
||||
// But if it's pruned, re-fetch from network again.
|
||||
if (hash != common.Hash{}) {
|
||||
if header := rawdb.ReadHeader(db, hash, number); header != nil {
|
||||
return header, nil
|
||||
}
|
||||
}
|
||||
if number >= chtCount*odr.IndexerConfig().ChtSize {
|
||||
// Retrieve the header via ODR, ensure the requested header is covered
|
||||
// by local trusted CHT.
|
||||
chts, _, chtHead := odr.ChtIndexer().Sections()
|
||||
if number >= chts*odr.IndexerConfig().ChtSize {
|
||||
return nil, errNoTrustedCht
|
||||
}
|
||||
r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number, Config: odr.IndexerConfig()}
|
||||
r := &ChtRequest{
|
||||
ChtRoot: GetChtRoot(db, chts-1, chtHead),
|
||||
ChtNum: chts - 1,
|
||||
BlockNum: number,
|
||||
Config: odr.IndexerConfig(),
|
||||
}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Header, nil
|
||||
}
|
||||
|
||||
// GetUntrustedHeaderByNumber fetches specified block header without correctness checking.
|
||||
// Note this function should only be used in light client checkpoint syncing.
|
||||
// GetUntrustedHeaderByNumber retrieves specified block header without
|
||||
// correctness checking. Note this function should only be used in light
|
||||
// client checkpoint syncing.
|
||||
func GetUntrustedHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64, peerId string) (*types.Header, error) {
|
||||
r := &ChtRequest{BlockNum: number, ChtNum: number / odr.IndexerConfig().ChtSize, Untrusted: true, PeerId: peerId, Config: odr.IndexerConfig()}
|
||||
// todo(rjl493456442) it's a hack to retrieve headers which is not covered
|
||||
// by CHT. Fix it in LES4
|
||||
r := &ChtRequest{
|
||||
BlockNum: number,
|
||||
ChtNum: number / odr.IndexerConfig().ChtSize,
|
||||
Untrusted: true,
|
||||
PeerId: peerId,
|
||||
Config: odr.IndexerConfig(),
|
||||
}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Header, nil
|
||||
}
|
||||
|
||||
// GetCanonicalHash retrieves the canonical block hash corresponding to the number.
|
||||
func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (common.Hash, error) {
|
||||
hash := rawdb.ReadCanonicalHash(odr.Database(), number)
|
||||
if (hash != common.Hash{}) {
|
||||
if hash != (common.Hash{}) {
|
||||
return hash, nil
|
||||
}
|
||||
header, err := GetHeaderByNumber(ctx, odr, number)
|
||||
if header != nil {
|
||||
return header.Hash(), nil
|
||||
if err != nil {
|
||||
return common.Hash{}, err
|
||||
}
|
||||
return common.Hash{}, err
|
||||
// number -> canonical mapping already be stored in db, get it.
|
||||
return header.Hash(), nil
|
||||
}
|
||||
|
||||
// GetTd retrieves the total difficulty corresponding to the number and hash.
|
||||
func GetTd(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*big.Int, error) {
|
||||
td := rawdb.ReadTd(odr.Database(), hash, number)
|
||||
if td != nil {
|
||||
return td, nil
|
||||
}
|
||||
_, err := GetHeaderByNumber(ctx, odr, number)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// <hash, number> -> td mapping already be stored in db, get it.
|
||||
return rawdb.ReadTd(odr.Database(), hash, number), nil
|
||||
}
|
||||
|
||||
// GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding.
|
||||
@ -96,15 +115,19 @@ func GetBodyRLP(ctx context.Context, odr OdrBackend, hash common.Hash, number ui
|
||||
if data := rawdb.ReadBodyRLP(odr.Database(), hash, number); data != nil {
|
||||
return data, nil
|
||||
}
|
||||
r := &BlockRequest{Hash: hash, Number: number}
|
||||
// Retrieve the block header first and pass it for verification.
|
||||
header, err := GetHeaderByNumber(ctx, odr, number)
|
||||
if err != nil {
|
||||
return nil, errNoHeader
|
||||
}
|
||||
r := &BlockRequest{Hash: hash, Number: number, Header: header}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return r.Rlp, nil
|
||||
}
|
||||
return r.Rlp, nil
|
||||
}
|
||||
|
||||
// GetBody retrieves the block body (transactons, uncles) corresponding to the
|
||||
// GetBody retrieves the block body (transactions, uncles) corresponding to the
|
||||
// hash.
|
||||
func GetBody(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*types.Body, error) {
|
||||
data, err := GetBodyRLP(ctx, odr, hash, number)
|
||||
@ -122,8 +145,8 @@ func GetBody(ctx context.Context, odr OdrBackend, hash common.Hash, number uint6
|
||||
// back from the stored header and body.
|
||||
func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*types.Block, error) {
|
||||
// Retrieve the block header and body contents
|
||||
header := rawdb.ReadHeader(odr.Database(), hash, number)
|
||||
if header == nil {
|
||||
header, err := GetHeaderByNumber(ctx, odr, number)
|
||||
if err != nil {
|
||||
return nil, errNoHeader
|
||||
}
|
||||
body, err := GetBody(ctx, odr, hash, number)
|
||||
@ -140,7 +163,11 @@ func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, num
|
||||
// Assume receipts are already stored locally and attempt to retrieve.
|
||||
receipts := rawdb.ReadRawReceipts(odr.Database(), hash, number)
|
||||
if receipts == nil {
|
||||
r := &ReceiptsRequest{Hash: hash, Number: number}
|
||||
header, err := GetHeaderByNumber(ctx, odr, number)
|
||||
if err != nil {
|
||||
return nil, errNoHeader
|
||||
}
|
||||
r := &ReceiptsRequest{Hash: hash, Number: number, Header: header}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -171,7 +198,6 @@ func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Return the logs without deriving any computed fields on the receipts
|
||||
logs := make([][]*types.Log, len(receipts))
|
||||
for i, receipt := range receipts {
|
||||
logs[i] = receipt.Logs
|
||||
@ -203,64 +229,51 @@ func GetUntrustedBlockLogs(ctx context.Context, odr OdrBackend, header *types.He
|
||||
return logs, nil
|
||||
}
|
||||
|
||||
// GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes
|
||||
func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxList []uint64) ([][]byte, error) {
|
||||
// GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to
|
||||
// the given bit index and section indexes.
|
||||
func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint64) ([][]byte, error) {
|
||||
var (
|
||||
db = odr.Database()
|
||||
result = make([][]byte, len(sectionIdxList))
|
||||
reqList []uint64
|
||||
reqIdx []int
|
||||
reqIndex []int
|
||||
reqSections []uint64
|
||||
db = odr.Database()
|
||||
result = make([][]byte, len(sections))
|
||||
)
|
||||
|
||||
var (
|
||||
bloomTrieCount, sectionHeadNum uint64
|
||||
sectionHead common.Hash
|
||||
)
|
||||
if odr.BloomTrieIndexer() != nil {
|
||||
bloomTrieCount, sectionHeadNum, sectionHead = odr.BloomTrieIndexer().Sections()
|
||||
canonicalHash := rawdb.ReadCanonicalHash(db, sectionHeadNum)
|
||||
// if the BloomTrie was injected as a trusted checkpoint, we have no canonical hash yet so we accept zero hash too
|
||||
for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) {
|
||||
bloomTrieCount--
|
||||
if bloomTrieCount > 0 {
|
||||
sectionHeadNum = bloomTrieCount*odr.IndexerConfig().BloomTrieSize - 1
|
||||
sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1)
|
||||
canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i, sectionIdx := range sectionIdxList {
|
||||
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*odr.IndexerConfig().BloomSize-1)
|
||||
// if we don't have the canonical hash stored for this section head number, we'll still look for
|
||||
// an entry with a zero sectionHead (we store it with zero section head too if we don't know it
|
||||
// at the time of the retrieval)
|
||||
bloomBits, err := rawdb.ReadBloomBits(db, bitIdx, sectionIdx, sectionHead)
|
||||
if err == nil {
|
||||
blooms, _, sectionHead := odr.BloomTrieIndexer().Sections()
|
||||
for i, section := range sections {
|
||||
sectionHead := rawdb.ReadCanonicalHash(db, (section+1)*odr.IndexerConfig().BloomSize-1)
|
||||
// If we don't have the canonical hash stored for this section head number,
|
||||
// we'll still look for an entry with a zero sectionHead (we store it with
|
||||
// zero section head too if we don't know it at the time of the retrieval)
|
||||
if bloomBits, _ := rawdb.ReadBloomBits(db, bit, section, sectionHead); len(bloomBits) != 0 {
|
||||
result[i] = bloomBits
|
||||
} else {
|
||||
// TODO(rjl493456442) Convert sectionIndex to BloomTrie relative index
|
||||
if sectionIdx >= bloomTrieCount {
|
||||
return nil, errNoTrustedBloomTrie
|
||||
}
|
||||
reqList = append(reqList, sectionIdx)
|
||||
reqIdx = append(reqIdx, i)
|
||||
continue
|
||||
}
|
||||
// TODO(rjl493456442) Convert sectionIndex to BloomTrie relative index
|
||||
if section >= blooms {
|
||||
return nil, errNoTrustedBloomTrie
|
||||
}
|
||||
reqSections = append(reqSections, section)
|
||||
reqIndex = append(reqIndex, i)
|
||||
}
|
||||
if reqList == nil {
|
||||
// Find all bloombits in database, nothing to query via odr, return.
|
||||
if reqSections == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1,
|
||||
BitIdx: bitIdx, SectionIndexList: reqList, Config: odr.IndexerConfig()}
|
||||
// Send odr request to retrieve missing bloombits.
|
||||
r := &BloomRequest{
|
||||
BloomTrieRoot: GetBloomTrieRoot(db, blooms-1, sectionHead),
|
||||
BloomTrieNum: blooms - 1,
|
||||
BitIdx: bit,
|
||||
SectionIndexList: reqSections,
|
||||
Config: odr.IndexerConfig(),
|
||||
}
|
||||
if err := odr.Retrieve(ctx, r); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
for i, idx := range reqIdx {
|
||||
result[idx] = r.BloomBits[i]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
for i, idx := range reqIndex {
|
||||
result[idx] = r.BloomBits[i]
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetTransaction retrieves a canonical transaction by hash and also returns its position in the chain
|
||||
@ -268,17 +281,16 @@ func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*t
|
||||
r := &TxStatusRequest{Hashes: []common.Hash{txHash}}
|
||||
if err := odr.Retrieve(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded {
|
||||
return nil, common.Hash{}, 0, 0, err
|
||||
} else {
|
||||
pos := r.Status[0].Lookup
|
||||
// first ensure that we have the header, otherwise block body retrieval will fail
|
||||
// also verify if this is a canonical block by getting the header by number and checking its hash
|
||||
if header, err := GetHeaderByNumber(ctx, odr, pos.BlockIndex); err != nil || header.Hash() != pos.BlockHash {
|
||||
return nil, common.Hash{}, 0, 0, err
|
||||
}
|
||||
if body, err := GetBody(ctx, odr, pos.BlockHash, pos.BlockIndex); err != nil || uint64(len(body.Transactions)) <= pos.Index || body.Transactions[pos.Index].Hash() != txHash {
|
||||
return nil, common.Hash{}, 0, 0, err
|
||||
} else {
|
||||
return body.Transactions[pos.Index], pos.BlockHash, pos.BlockIndex, pos.Index, nil
|
||||
}
|
||||
}
|
||||
pos := r.Status[0].Lookup
|
||||
// first ensure that we have the header, otherwise block body retrieval will fail
|
||||
// also verify if this is a canonical block by getting the header by number and checking its hash
|
||||
if header, err := GetHeaderByNumber(ctx, odr, pos.BlockIndex); err != nil || header.Hash() != pos.BlockHash {
|
||||
return nil, common.Hash{}, 0, 0, err
|
||||
}
|
||||
body, err := GetBody(ctx, odr, pos.BlockHash, pos.BlockIndex)
|
||||
if err != nil || uint64(len(body.Transactions)) <= pos.Index || body.Transactions[pos.Index].Hash() != txHash {
|
||||
return nil, common.Hash{}, 0, 0, err
|
||||
}
|
||||
return body.Transactions[pos.Index], pos.BlockHash, pos.BlockIndex, pos.Index, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user