core, cmd, vendor: fixes and database inspection tool (#15)
* core, eth: some fixes for freezer * vendor, core/rawdb, cmd/geth: add db inspector * core, cmd/utils: check ancient store path forceily * cmd/geth, common, core/rawdb: a few fixes * cmd/geth: support windows file rename and fix rename error * core: support ancient plugin * core, cmd: streaming file copy * cmd, consensus, core, tests: keep genesis in leveldb * core: write txlookup during ancient init * core: bump database version
This commit is contained in:
gary rong
committed by
Péter Szilágyi
Péter Szilágyi
parent
42c746d6f4
commit
37d280da41
@ -89,7 +89,16 @@ func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) *uint64 {
|
||||
return &number
|
||||
}
|
||||
|
||||
// DeleteHeaderNumber removes hash to number mapping.
|
||||
// WriteHeaderNumber stores the hash->number mapping.
|
||||
func WriteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
|
||||
key := headerNumberKey(hash)
|
||||
enc := encodeBlockNumber(number)
|
||||
if err := db.Put(key, enc); err != nil {
|
||||
log.Crit("Failed to store hash to number mapping", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteHeaderNumber removes hash->number mapping.
|
||||
func DeleteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash) {
|
||||
if err := db.Delete(headerNumberKey(hash)); err != nil {
|
||||
log.Crit("Failed to delete hash to number mapping", "err", err)
|
||||
@ -206,22 +215,19 @@ func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header
|
||||
// WriteHeader stores a block header into the database and also stores the hash-
|
||||
// to-number mapping.
|
||||
func WriteHeader(db ethdb.KeyValueWriter, header *types.Header) {
|
||||
// Write the hash -> number mapping
|
||||
var (
|
||||
hash = header.Hash()
|
||||
number = header.Number.Uint64()
|
||||
encoded = encodeBlockNumber(number)
|
||||
hash = header.Hash()
|
||||
number = header.Number.Uint64()
|
||||
)
|
||||
key := headerNumberKey(hash)
|
||||
if err := db.Put(key, encoded); err != nil {
|
||||
log.Crit("Failed to store hash to number mapping", "err", err)
|
||||
}
|
||||
// Write the hash -> number mapping
|
||||
WriteHeaderNumber(db, hash, number)
|
||||
|
||||
// Write the encoded header
|
||||
data, err := rlp.EncodeToBytes(header)
|
||||
if err != nil {
|
||||
log.Crit("Failed to RLP encode header", "err", err)
|
||||
}
|
||||
key = headerKey(number, hash)
|
||||
key := headerKey(number, hash)
|
||||
if err := db.Put(key, data); err != nil {
|
||||
log.Crit("Failed to store header", "err", err)
|
||||
}
|
||||
|
||||
@ -80,6 +80,20 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha
|
||||
}
|
||||
}
|
||||
|
||||
// ReadAncientPath retrieves ancient database path which is recorded during the
|
||||
// first node setup or forcibly changed by user.
|
||||
func ReadAncientPath(db ethdb.KeyValueReader) string {
|
||||
data, _ := db.Get(ancientKey)
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// WriteAncientPath writes ancient database path into the key-value database.
|
||||
func WriteAncientPath(db ethdb.KeyValueWriter, path string) {
|
||||
if err := db.Put(ancientKey, []byte(path)); err != nil {
|
||||
log.Crit("Failed to store ancient path", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadPreimage retrieves a single preimage of the provided hash.
|
||||
func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
|
||||
data, _ := db.Get(preimageKey(hash))
|
||||
|
||||
@ -17,11 +17,17 @@
|
||||
package rawdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/leveldb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
)
|
||||
|
||||
// freezerdb is a database wrapper that enabled freezer data retrievals.
|
||||
@ -66,6 +72,11 @@ func (db *nofreezedb) Ancients() (uint64, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
// AncientSize returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AncientSize(kind string) (uint64, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
// AppendAncient returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error {
|
||||
return errNotSupported
|
||||
@ -140,5 +151,128 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
|
||||
kvdb.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Make sure we always use the same ancient store.
|
||||
//
|
||||
// | stored == nil | stored != nil
|
||||
// ----------------+------------------+----------------------
|
||||
// freezer == nil | non-freezer mode | ancient store missing
|
||||
// freezer != nil | initialize | ensure consistency
|
||||
stored := ReadAncientPath(kvdb)
|
||||
if stored == "" && freezer != "" {
|
||||
WriteAncientPath(kvdb, freezer)
|
||||
} else if stored != freezer {
|
||||
log.Warn("Ancient path mismatch", "stored", stored, "given", freezer)
|
||||
log.Crit("Please use a consistent ancient path or migrate it via the command line tool `geth migrate-ancient`")
|
||||
}
|
||||
return frdb, nil
|
||||
}
|
||||
|
||||
// InspectDatabase traverses the entire database and checks the size
|
||||
// of all different categories of data.
|
||||
func InspectDatabase(db ethdb.Database) error {
|
||||
it := db.NewIterator()
|
||||
defer it.Release()
|
||||
|
||||
var (
|
||||
count int64
|
||||
start = time.Now()
|
||||
logged = time.Now()
|
||||
|
||||
// Key-value store statistics
|
||||
total common.StorageSize
|
||||
headerSize common.StorageSize
|
||||
bodySize common.StorageSize
|
||||
receiptSize common.StorageSize
|
||||
tdSize common.StorageSize
|
||||
numHashPairing common.StorageSize
|
||||
hashNumPairing common.StorageSize
|
||||
trieSize common.StorageSize
|
||||
txlookupSize common.StorageSize
|
||||
preimageSize common.StorageSize
|
||||
bloomBitsSize common.StorageSize
|
||||
|
||||
// Ancient store statistics
|
||||
ancientHeaders common.StorageSize
|
||||
ancientBodies common.StorageSize
|
||||
ancientReceipts common.StorageSize
|
||||
ancientHashes common.StorageSize
|
||||
ancientTds common.StorageSize
|
||||
|
||||
// Les statistic
|
||||
ChtTrieNodes common.StorageSize
|
||||
BloomTrieNodes common.StorageSize
|
||||
)
|
||||
// Inspect key-value database first.
|
||||
for it.Next() {
|
||||
var (
|
||||
key = it.Key()
|
||||
size = common.StorageSize(len(key) + len(it.Value()))
|
||||
)
|
||||
total += size
|
||||
switch {
|
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
|
||||
tdSize += size
|
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
|
||||
numHashPairing += size
|
||||
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
|
||||
headerSize += size
|
||||
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
|
||||
hashNumPairing += size
|
||||
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
|
||||
bodySize += size
|
||||
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
|
||||
receiptSize += size
|
||||
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
|
||||
txlookupSize += size
|
||||
case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
|
||||
preimageSize += size
|
||||
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
|
||||
bloomBitsSize += size
|
||||
case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength:
|
||||
ChtTrieNodes += size
|
||||
case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength:
|
||||
BloomTrieNodes += size
|
||||
case len(key) == common.HashLength:
|
||||
trieSize += size
|
||||
}
|
||||
count += 1
|
||||
if count%1000 == 0 && time.Since(logged) > 8*time.Second {
|
||||
log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
// Inspect append-only file store then.
|
||||
ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds}
|
||||
for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} {
|
||||
if size, err := db.AncientSize(category); err == nil {
|
||||
*ancients[i] += common.StorageSize(size)
|
||||
total += common.StorageSize(size)
|
||||
}
|
||||
}
|
||||
// Display the database statistic.
|
||||
stats := [][]string{
|
||||
{"Key-Value store", "Headers", headerSize.String()},
|
||||
{"Key-Value store", "Bodies", bodySize.String()},
|
||||
{"Key-Value store", "Receipts", receiptSize.String()},
|
||||
{"Key-Value store", "Difficulties", tdSize.String()},
|
||||
{"Key-Value store", "Block number->hash", numHashPairing.String()},
|
||||
{"Key-Value store", "Block hash->number", hashNumPairing.String()},
|
||||
{"Key-Value store", "Transaction index", txlookupSize.String()},
|
||||
{"Key-Value store", "Bloombit index", bloomBitsSize.String()},
|
||||
{"Key-Value store", "Trie nodes", trieSize.String()},
|
||||
{"Key-Value store", "Trie preimages", preimageSize.String()},
|
||||
{"Ancient store", "Headers", ancientHeaders.String()},
|
||||
{"Ancient store", "Bodies", ancientBodies.String()},
|
||||
{"Ancient store", "Receipts", ancientReceipts.String()},
|
||||
{"Ancient store", "Difficulties", ancientTds.String()},
|
||||
{"Ancient store", "Block number->hash", ancientHashes.String()},
|
||||
{"Light client", "CHT trie nodes", ChtTrieNodes.String()},
|
||||
{"Light client", "Bloom trie nodes", BloomTrieNodes.String()},
|
||||
}
|
||||
table := tablewriter.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"Database", "Category", "Size"})
|
||||
table.SetFooter([]string{"", "Total", total.String()})
|
||||
table.AppendBulk(stats)
|
||||
table.Render()
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@ -39,6 +40,10 @@ var (
|
||||
// errOutOrderInsertion is returned if the user attempts to inject out-of-order
|
||||
// binary blobs into the freezer.
|
||||
errOutOrderInsertion = errors.New("the append operation is out-order")
|
||||
|
||||
// errSymlinkDatadir is returned if the ancient directory specified by user
|
||||
// is a symbolic link.
|
||||
errSymlinkDatadir = errors.New("symbolic link datadir is not supported")
|
||||
)
|
||||
|
||||
const (
|
||||
@ -78,6 +83,13 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
|
||||
readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
|
||||
writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil)
|
||||
)
|
||||
// Ensure the datadir is not a symbolic link if it exists.
|
||||
if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
log.Warn("Symbolic link ancient database is not supported", "path", datadir)
|
||||
return nil, errSymlinkDatadir
|
||||
}
|
||||
}
|
||||
// Leveldb uses LOCK as the filelock filename. To prevent the
|
||||
// name collision, we use FLOCK as the lock name.
|
||||
lock, _, err := fileutil.Flock(filepath.Join(datadir, "FLOCK"))
|
||||
@ -107,6 +119,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
|
||||
lock.Release()
|
||||
return nil, err
|
||||
}
|
||||
log.Info("Opened ancient database", "database", datadir)
|
||||
return freezer, nil
|
||||
}
|
||||
|
||||
@ -149,6 +162,14 @@ func (f *freezer) Ancients() (uint64, error) {
|
||||
return atomic.LoadUint64(&f.frozen), nil
|
||||
}
|
||||
|
||||
// AncientSize returns the ancient size of the specified category.
|
||||
func (f *freezer) AncientSize(kind string) (uint64, error) {
|
||||
if table := f.tables[kind]; table != nil {
|
||||
return table.size()
|
||||
}
|
||||
return 0, errUnknownTable
|
||||
}
|
||||
|
||||
// AppendAncient injects all binary blobs belong to block at the end of the
|
||||
// append-only immutable table files.
|
||||
//
|
||||
|
||||
@ -515,6 +515,19 @@ func (t *freezerTable) has(number uint64) bool {
|
||||
return atomic.LoadUint64(&t.items) > number
|
||||
}
|
||||
|
||||
// size returns the total data size in the freezer table.
|
||||
func (t *freezerTable) size() (uint64, error) {
|
||||
t.lock.RLock()
|
||||
defer t.lock.RUnlock()
|
||||
|
||||
stat, err := t.index.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total := uint64(t.maxFileSize)*uint64(t.headId-t.tailId) + uint64(t.headBytes) + uint64(stat.Size())
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// Sync pushes any pending data from memory out to disk. This is an expensive
|
||||
// operation, so use it with care.
|
||||
func (t *freezerTable) Sync() error {
|
||||
|
||||
@ -41,6 +41,9 @@ var (
|
||||
// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
|
||||
fastTrieProgressKey = []byte("TrieSync")
|
||||
|
||||
// ancientKey tracks the absolute path of ancient database.
|
||||
ancientKey = []byte("AncientPath")
|
||||
|
||||
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
|
||||
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
|
||||
headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
|
||||
|
||||
@ -68,6 +68,12 @@ func (t *table) Ancients() (uint64, error) {
|
||||
return t.db.Ancients()
|
||||
}
|
||||
|
||||
// AncientSize is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) AncientSize(kind string) (uint64, error) {
|
||||
return t.db.AncientSize(kind)
|
||||
}
|
||||
|
||||
// AppendAncient is a noop passthrough that just forwards the request to the underlying
|
||||
// database.
|
||||
func (t *table) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error {
|
||||
|
||||
Reference in New Issue
Block a user