cmd, core, eth: background transaction indexing (#20302)
* cmd, core, eth: init tx lookup in background * core/rawdb: tiny log fixes to make it clearer what's happening * core, eth: fix rebase errors * core/rawdb: make reindexing less generic, but more optimal * rlp: implement rlp list iterator * core/rawdb: new implementation of tx indexing/unindex using generic tx iterator and hashing rlp-data * core/rawdb, cmd/utils: fix review concerns * cmd/utils: fix merge issue * core/rawdb: add some log formatting polishes Co-authored-by: rjl493456442 <garyrong0905@gmail.com> Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
Martin Holst Swende
committed by
GitHub
GitHub
parent
6f54ae24cd
commit
4535230059
@ -172,6 +172,43 @@ func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) {
|
||||
}
|
||||
}
|
||||
|
||||
// ReadTxIndexTail retrieves the number of oldest indexed block
|
||||
// whose transaction indices has been indexed. If the corresponding entry
|
||||
// is non-existent in database it means the indexing has been finished.
|
||||
func ReadTxIndexTail(db ethdb.KeyValueReader) *uint64 {
|
||||
data, _ := db.Get(txIndexTailKey)
|
||||
if len(data) != 8 {
|
||||
return nil
|
||||
}
|
||||
number := binary.BigEndian.Uint64(data)
|
||||
return &number
|
||||
}
|
||||
|
||||
// WriteTxIndexTail stores the number of oldest indexed block
|
||||
// into database.
|
||||
func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) {
|
||||
if err := db.Put(txIndexTailKey, encodeBlockNumber(number)); err != nil {
|
||||
log.Crit("Failed to store the transaction index tail", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadFastTxLookupLimit retrieves the tx lookup limit used in fast sync.
|
||||
func ReadFastTxLookupLimit(db ethdb.KeyValueReader) *uint64 {
|
||||
data, _ := db.Get(fastTxLookupLimitKey)
|
||||
if len(data) != 8 {
|
||||
return nil
|
||||
}
|
||||
number := binary.BigEndian.Uint64(data)
|
||||
return &number
|
||||
}
|
||||
|
||||
// WriteFastTxLookupLimit stores the txlookup limit used in fast sync into database.
|
||||
func WriteFastTxLookupLimit(db ethdb.KeyValueWriter, number uint64) {
|
||||
if err := db.Put(fastTxLookupLimitKey, encodeBlockNumber(number)); err != nil {
|
||||
log.Crit("Failed to store transaction lookup limit for fast sync", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadHeaderRLP retrieves a block header in its raw RLP database encoding.
|
||||
func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
|
||||
// First try to look up the data in ancient database. Extra hash
|
||||
@ -290,6 +327,25 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
|
||||
return nil // Can't find the data anywhere.
|
||||
}
|
||||
|
||||
// ReadCanonicalBodyRLP retrieves the block body (transactions and uncles) for the canonical
|
||||
// block at number, in RLP encoding.
|
||||
func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
|
||||
// If it's an ancient one, we don't need the canonical hash
|
||||
data, _ := db.Ancient(freezerBodiesTable, number)
|
||||
if len(data) == 0 {
|
||||
// Need to get the hash
|
||||
data, _ = db.Get(blockBodyKey(number, ReadCanonicalHash(db, number)))
|
||||
// In the background freezer is moving data from leveldb to flatten files.
|
||||
// So during the first check for ancient db, the data is not yet in there,
|
||||
// but when we reach into leveldb, the data was already moved. That would
|
||||
// result in a not found error.
|
||||
if len(data) == 0 {
|
||||
data, _ = db.Ancient(freezerBodiesTable, number)
|
||||
}
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// WriteBodyRLP stores an RLP encoded block body into the database.
|
||||
func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp rlp.RawValue) {
|
||||
if err := db.Put(blockBodyKey(number, hash), rlp); err != nil {
|
||||
|
||||
@ -63,9 +63,31 @@ func WriteTxLookupEntries(db ethdb.KeyValueWriter, block *types.Block) {
|
||||
}
|
||||
}
|
||||
|
||||
// WriteTxLookupEntriesByHash is identical to WriteTxLookupEntries, but does not
|
||||
// require a full types.Block as input.
|
||||
func WriteTxLookupEntriesByHash(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) {
|
||||
numberBytes := new(big.Int).SetUint64(number).Bytes()
|
||||
for _, hash := range hashes {
|
||||
if err := db.Put(txLookupKey(hash), numberBytes); err != nil {
|
||||
log.Crit("Failed to store transaction lookup entry", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteTxLookupEntry removes all transaction data associated with a hash.
|
||||
func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) {
|
||||
db.Delete(txLookupKey(hash))
|
||||
if err := db.Delete(txLookupKey(hash)); err != nil {
|
||||
log.Crit("Failed to delete transaction lookup entry", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteTxLookupEntries removes all transaction lookups for a given block.
|
||||
func DeleteTxLookupEntriesByHash(db ethdb.KeyValueWriter, hashes []common.Hash) {
|
||||
for _, hash := range hashes {
|
||||
if err := db.Delete(txLookupKey(hash)); err != nil {
|
||||
log.Crit("Failed to delete transaction lookup entry", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ReadTransaction retrieves a specific transaction from the database, along with
|
||||
|
||||
305
core/rawdb/chain_iterator.go
Normal file
305
core/rawdb/chain_iterator.go
Normal file
@ -0,0 +1,305 @@
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb
|
||||
|
||||
import (
|
||||
"math"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/prque"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"golang.org/x/crypto/sha3"
|
||||
)
|
||||
|
||||
// InitDatabaseFromFreezer reinitializes an empty database from a previous batch
|
||||
// of frozen ancient blocks. The method iterates over all the frozen blocks and
|
||||
// injects into the database the block hash->number mappings.
|
||||
func InitDatabaseFromFreezer(db ethdb.Database) {
|
||||
// If we can't access the freezer or it's empty, abort
|
||||
frozen, err := db.Ancients()
|
||||
if err != nil || frozen == 0 {
|
||||
return
|
||||
}
|
||||
var (
|
||||
batch = db.NewBatch()
|
||||
start = time.Now()
|
||||
logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
|
||||
hash common.Hash
|
||||
)
|
||||
for i := uint64(0); i < frozen; i++ {
|
||||
// Since the freezer has all data in sequential order on a file,
|
||||
// it would be 'neat' to read more data in one go, and let the
|
||||
// freezerdb return N items (e.g up to 1000 items per go)
|
||||
// That would require an API change in Ancients though
|
||||
if h, err := db.Ancient(freezerHashTable, i); err != nil {
|
||||
log.Crit("Failed to init database from freezer", "err", err)
|
||||
} else {
|
||||
hash = common.BytesToHash(h)
|
||||
}
|
||||
WriteHeaderNumber(batch, hash, i)
|
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
if batch.ValueSize() > ethdb.IdealBatchSize {
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed to write data to db", "err", err)
|
||||
}
|
||||
batch.Reset()
|
||||
}
|
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second {
|
||||
log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed to write data to db", "err", err)
|
||||
}
|
||||
batch.Reset()
|
||||
|
||||
WriteHeadHeaderHash(db, hash)
|
||||
WriteHeadFastBlockHash(db, hash)
|
||||
log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
}
|
||||
|
||||
type blockTxHashes struct {
|
||||
number uint64
|
||||
hashes []common.Hash
|
||||
}
|
||||
|
||||
// iterateTransactions iterates over all transactions in the (canon) block
|
||||
// number(s) given, and yields the hashes on a channel
|
||||
func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool) (chan *blockTxHashes, chan struct{}) {
|
||||
// One thread sequentially reads data from db
|
||||
type numberRlp struct {
|
||||
number uint64
|
||||
rlp rlp.RawValue
|
||||
}
|
||||
if to == from {
|
||||
return nil, nil
|
||||
}
|
||||
threads := to - from
|
||||
if cpus := runtime.NumCPU(); threads > uint64(cpus) {
|
||||
threads = uint64(cpus)
|
||||
}
|
||||
var (
|
||||
rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel
|
||||
hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
|
||||
abortCh = make(chan struct{})
|
||||
)
|
||||
// lookup runs in one instance
|
||||
lookup := func() {
|
||||
n, end := from, to
|
||||
if reverse {
|
||||
n, end = to-1, from-1
|
||||
}
|
||||
defer close(rlpCh)
|
||||
for n != end {
|
||||
data := ReadCanonicalBodyRLP(db, n)
|
||||
// Feed the block to the aggregator, or abort on interrupt
|
||||
select {
|
||||
case rlpCh <- &numberRlp{n, data}:
|
||||
case <-abortCh:
|
||||
return
|
||||
}
|
||||
if reverse {
|
||||
n--
|
||||
} else {
|
||||
n++
|
||||
}
|
||||
}
|
||||
}
|
||||
// process runs in parallell
|
||||
nThreadsAlive := int32(threads)
|
||||
process := func() {
|
||||
defer func() {
|
||||
// Last processor closes the result channel
|
||||
if atomic.AddInt32(&nThreadsAlive, -1) == 0 {
|
||||
close(hashesCh)
|
||||
}
|
||||
}()
|
||||
|
||||
var hasher = sha3.NewLegacyKeccak256()
|
||||
for data := range rlpCh {
|
||||
it, err := rlp.NewListIterator(data.rlp)
|
||||
if err != nil {
|
||||
log.Warn("tx iteration error", "error", err)
|
||||
return
|
||||
}
|
||||
it.Next()
|
||||
txs := it.Value()
|
||||
txIt, err := rlp.NewListIterator(txs)
|
||||
if err != nil {
|
||||
log.Warn("tx iteration error", "error", err)
|
||||
return
|
||||
}
|
||||
var hashes []common.Hash
|
||||
for txIt.Next() {
|
||||
if err := txIt.Err(); err != nil {
|
||||
log.Warn("tx iteration error", "error", err)
|
||||
return
|
||||
}
|
||||
var txHash common.Hash
|
||||
hasher.Reset()
|
||||
hasher.Write(txIt.Value())
|
||||
hasher.Sum(txHash[:0])
|
||||
hashes = append(hashes, txHash)
|
||||
}
|
||||
result := &blockTxHashes{
|
||||
hashes: hashes,
|
||||
number: data.number,
|
||||
}
|
||||
// Feed the block to the aggregator, or abort on interrupt
|
||||
select {
|
||||
case hashesCh <- result:
|
||||
case <-abortCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
go lookup() // start the sequential db accessor
|
||||
for i := 0; i < int(threads); i++ {
|
||||
go process()
|
||||
}
|
||||
return hashesCh, abortCh
|
||||
}
|
||||
|
||||
// IndexTransactions creates txlookup indices of the specified block range.
|
||||
//
|
||||
// This function iterates canonical chain in reverse order, it has one main advantage:
|
||||
// We can write tx index tail flag periodically even without the whole indexing
|
||||
// procedure is finished. So that we can resume indexing procedure next time quickly.
|
||||
func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
|
||||
// short circuit for invalid range
|
||||
if from >= to {
|
||||
return
|
||||
}
|
||||
var (
|
||||
hashesCh, abortCh = iterateTransactions(db, from, to, true)
|
||||
batch = db.NewBatch()
|
||||
start = time.Now()
|
||||
logged = start.Add(-7 * time.Second)
|
||||
// Since we iterate in reverse, we expect the first number to come
|
||||
// in to be [to-1]. Therefore, setting lastNum to means that the
|
||||
// prqueue gap-evaluation will work correctly
|
||||
lastNum = to
|
||||
queue = prque.New(nil)
|
||||
// for stats reporting
|
||||
blocks, txs = 0, 0
|
||||
)
|
||||
defer close(abortCh)
|
||||
|
||||
for chanDelivery := range hashesCh {
|
||||
// Push the delivery into the queue and process contiguous ranges.
|
||||
// Since we iterate in reverse, so lower numbers have lower prio, and
|
||||
// we can use the number directly as prio marker
|
||||
queue.Push(chanDelivery, int64(chanDelivery.number))
|
||||
for !queue.Empty() {
|
||||
// If the next available item is gapped, return
|
||||
if _, priority := queue.Peek(); priority != int64(lastNum-1) {
|
||||
break
|
||||
}
|
||||
// Next block available, pop it off and index it
|
||||
delivery := queue.PopItem().(*blockTxHashes)
|
||||
lastNum = delivery.number
|
||||
WriteTxLookupEntriesByHash(batch, delivery.number, delivery.hashes)
|
||||
blocks++
|
||||
txs += len(delivery.hashes)
|
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
if batch.ValueSize() > ethdb.IdealBatchSize {
|
||||
// Also write the tail there
|
||||
WriteTxIndexTail(batch, lastNum)
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed writing batch to db", "error", err)
|
||||
return
|
||||
}
|
||||
batch.Reset()
|
||||
}
|
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second {
|
||||
log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
}
|
||||
if lastNum < to {
|
||||
WriteTxIndexTail(batch, lastNum)
|
||||
// No need to write the batch if we never entered the loop above...
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed writing batch to db", "error", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
}
|
||||
|
||||
// UnindexTransactions removes txlookup indices of the specified block range.
|
||||
func UnindexTransactions(db ethdb.Database, from uint64, to uint64) {
|
||||
// short circuit for invalid range
|
||||
if from >= to {
|
||||
return
|
||||
}
|
||||
// Write flag first and then unindex the transaction indices. Some indices
|
||||
// will be left in the database if crash happens but it's fine.
|
||||
WriteTxIndexTail(db, to)
|
||||
// If only one block is unindexed, do it directly
|
||||
//if from+1 == to {
|
||||
// data := ReadCanonicalBodyRLP(db, uint64(from))
|
||||
// DeleteTxLookupEntries(db, ReadBlock(db, ReadCanonicalHash(db, from), from))
|
||||
// log.Info("Unindexed transactions", "blocks", 1, "tail", to)
|
||||
// return
|
||||
//}
|
||||
// TODO @holiman, add this back (if we want it)
|
||||
var (
|
||||
hashesCh, abortCh = iterateTransactions(db, from, to, false)
|
||||
batch = db.NewBatch()
|
||||
start = time.Now()
|
||||
logged = start.Add(-7 * time.Second)
|
||||
)
|
||||
defer close(abortCh)
|
||||
// Otherwise spin up the concurrent iterator and unindexer
|
||||
blocks, txs := 0, 0
|
||||
for delivery := range hashesCh {
|
||||
DeleteTxLookupEntriesByHash(batch, delivery.hashes)
|
||||
txs += len(delivery.hashes)
|
||||
blocks++
|
||||
|
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
// A batch counts the size of deletion as '1', so we need to flush more
|
||||
// often than that.
|
||||
if blocks%1000 == 0 {
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed writing batch to db", "error", err)
|
||||
return
|
||||
}
|
||||
batch.Reset()
|
||||
}
|
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second {
|
||||
log.Info("Unindexing transactions", "blocks", "txs", txs, int64(math.Abs(float64(delivery.number-from))), "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed writing batch to db", "error", err)
|
||||
return
|
||||
}
|
||||
log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
}
|
||||
82
core/rawdb/chain_iterator_test.go
Normal file
82
core/rawdb/chain_iterator_test.go
Normal file
@ -0,0 +1,82 @@
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
)
|
||||
|
||||
func TestChainIterator(t *testing.T) {
|
||||
// Construct test chain db
|
||||
chainDb := NewMemoryDatabase()
|
||||
|
||||
var block *types.Block
|
||||
var txs []*types.Transaction
|
||||
for i := uint64(0); i <= 10; i++ {
|
||||
if i == 0 {
|
||||
block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, nil, nil, nil) // Empty genesis block
|
||||
} else {
|
||||
tx := types.NewTransaction(i, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
|
||||
txs = append(txs, tx)
|
||||
block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil)
|
||||
}
|
||||
WriteBlock(chainDb, block)
|
||||
WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
|
||||
}
|
||||
|
||||
var cases = []struct {
|
||||
from, to uint64
|
||||
reverse bool
|
||||
expect []int
|
||||
}{
|
||||
{0, 11, true, []int{10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}},
|
||||
{0, 0, true, nil},
|
||||
{0, 5, true, []int{4, 3, 2, 1, 0}},
|
||||
{10, 11, true, []int{10}},
|
||||
{0, 11, false, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
|
||||
{0, 0, false, nil},
|
||||
{10, 11, false, []int{10}},
|
||||
}
|
||||
for i, c := range cases {
|
||||
var numbers []int
|
||||
hashCh, _ := iterateTransactions(chainDb, c.from, c.to, c.reverse)
|
||||
if hashCh != nil {
|
||||
for h := range hashCh {
|
||||
numbers = append(numbers, int(h.number))
|
||||
if len(h.hashes) > 0 {
|
||||
if got, exp := h.hashes[0], txs[h.number-1].Hash(); got != exp {
|
||||
t.Fatalf("hash wrong, got %x exp %x", got, exp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !c.reverse {
|
||||
sort.Ints(numbers)
|
||||
} else {
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(numbers)))
|
||||
}
|
||||
if !reflect.DeepEqual(numbers, c.expect) {
|
||||
t.Fatalf("Case %d failed, visit element mismatch, want %v, got %v", i, c.expect, numbers)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,127 +0,0 @@
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package rawdb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/prque"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
)
|
||||
|
||||
// InitDatabaseFromFreezer reinitializes an empty database from a previous batch
|
||||
// of frozen ancient blocks. The method iterates over all the frozen blocks and
|
||||
// injects into the database the block hash->number mappings and the transaction
|
||||
// lookup entries.
|
||||
func InitDatabaseFromFreezer(db ethdb.Database) error {
|
||||
// If we can't access the freezer or it's empty, abort
|
||||
frozen, err := db.Ancients()
|
||||
if err != nil || frozen == 0 {
|
||||
return err
|
||||
}
|
||||
// Blocks previously frozen, iterate over- and hash them concurrently
|
||||
var (
|
||||
number = ^uint64(0) // -1
|
||||
results = make(chan *types.Block, 4*runtime.NumCPU())
|
||||
)
|
||||
abort := make(chan struct{})
|
||||
defer close(abort)
|
||||
|
||||
for i := 0; i < runtime.NumCPU(); i++ {
|
||||
go func() {
|
||||
for {
|
||||
// Fetch the next task number, terminating if everything's done
|
||||
n := atomic.AddUint64(&number, 1)
|
||||
if n >= frozen {
|
||||
return
|
||||
}
|
||||
// Retrieve the block from the freezer. If successful, pre-cache
|
||||
// the block hash and the individual transaction hashes for storing
|
||||
// into the database.
|
||||
block := ReadBlock(db, ReadCanonicalHash(db, n), n)
|
||||
if block != nil {
|
||||
block.Hash()
|
||||
for _, tx := range block.Transactions() {
|
||||
tx.Hash()
|
||||
}
|
||||
}
|
||||
// Feed the block to the aggregator, or abort on interrupt
|
||||
select {
|
||||
case results <- block:
|
||||
case <-abort:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
// Reassemble the blocks into a contiguous stream and push them out to disk
|
||||
var (
|
||||
queue = prque.New(nil)
|
||||
next = int64(0)
|
||||
|
||||
batch = db.NewBatch()
|
||||
start = time.Now()
|
||||
logged time.Time
|
||||
)
|
||||
for i := uint64(0); i < frozen; i++ {
|
||||
// Retrieve the next result and bail if it's nil
|
||||
block := <-results
|
||||
if block == nil {
|
||||
return errors.New("broken ancient database")
|
||||
}
|
||||
// Push the block into the import queue and process contiguous ranges
|
||||
queue.Push(block, -int64(block.NumberU64()))
|
||||
for !queue.Empty() {
|
||||
// If the next available item is gapped, return
|
||||
if _, priority := queue.Peek(); -priority != next {
|
||||
break
|
||||
}
|
||||
// Next block available, pop it off and index it
|
||||
block = queue.PopItem().(*types.Block)
|
||||
next++
|
||||
|
||||
// Inject hash<->number mapping and txlookup indexes
|
||||
WriteHeaderNumber(batch, block.Hash(), block.NumberU64())
|
||||
WriteTxLookupEntries(batch, block)
|
||||
|
||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||
if batch.ValueSize() > ethdb.IdealBatchSize || uint64(next) == frozen {
|
||||
if err := batch.Write(); err != nil {
|
||||
return err
|
||||
}
|
||||
batch.Reset()
|
||||
}
|
||||
// If we've spent too much time already, notify the user of what we're doing
|
||||
if time.Since(logged) > 8*time.Second {
|
||||
log.Info("Initializing chain from ancient data", "number", block.Number(), "hash", block.Hash(), "total", frozen-1, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
}
|
||||
hash := ReadCanonicalHash(db, frozen-1)
|
||||
WriteHeadHeaderHash(db, hash)
|
||||
WriteHeadFastBlockHash(db, hash)
|
||||
|
||||
log.Info("Initialized chain from ancient data", "number", frozen-1, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
return nil
|
||||
}
|
||||
@ -47,6 +47,12 @@ var (
|
||||
// snapshotJournalKey tracks the in-memory diff layers across restarts.
|
||||
snapshotJournalKey = []byte("SnapshotJournal")
|
||||
|
||||
// txIndexTailKey tracks the oldest block whose transactions have been indexed.
|
||||
txIndexTailKey = []byte("TransactionIndexTail")
|
||||
|
||||
// fastTxLookupLimitKey tracks the transaction lookup limit during fast sync.
|
||||
fastTxLookupLimitKey = []byte("FastTransactionLookupLimit")
|
||||
|
||||
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
|
||||
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
|
||||
headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
|
||||
|
||||
Reference in New Issue
Block a user