core, cmd, vendor: fixes and database inspection tool (#15)
* core, eth: some fixes for freezer * vendor, core/rawdb, cmd/geth: add db inspector * core, cmd/utils: check ancient store path forceily * cmd/geth, common, core/rawdb: a few fixes * cmd/geth: support windows file rename and fix rename error * core: support ancient plugin * core, cmd: streaming file copy * cmd, consensus, core, tests: keep genesis in leveldb * core: write txlookup during ancient init * core: bump database version
This commit is contained in:
gary rong
committed by
Péter Szilágyi
Péter Szilágyi
parent
42c746d6f4
commit
37d280da41
@ -17,11 +17,17 @@
|
||||
package rawdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/leveldb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
)
|
||||
|
||||
// freezerdb is a database wrapper that enabled freezer data retrievals.
|
||||
@ -66,6 +72,11 @@ func (db *nofreezedb) Ancients() (uint64, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
// AncientSize returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AncientSize(kind string) (uint64, error) {
|
||||
return 0, errNotSupported
|
||||
}
|
||||
|
||||
// AppendAncient returns an error as we don't have a backing chain freezer.
|
||||
func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error {
|
||||
return errNotSupported
|
||||
@ -140,5 +151,128 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
|
||||
kvdb.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Make sure we always use the same ancient store.
|
||||
//
|
||||
// | stored == nil | stored != nil
|
||||
// ----------------+------------------+----------------------
|
||||
// freezer == nil | non-freezer mode | ancient store missing
|
||||
// freezer != nil | initialize | ensure consistency
|
||||
stored := ReadAncientPath(kvdb)
|
||||
if stored == "" && freezer != "" {
|
||||
WriteAncientPath(kvdb, freezer)
|
||||
} else if stored != freezer {
|
||||
log.Warn("Ancient path mismatch", "stored", stored, "given", freezer)
|
||||
log.Crit("Please use a consistent ancient path or migrate it via the command line tool `geth migrate-ancient`")
|
||||
}
|
||||
return frdb, nil
|
||||
}
|
||||
|
||||
// InspectDatabase traverses the entire database and checks the size
|
||||
// of all different categories of data.
|
||||
func InspectDatabase(db ethdb.Database) error {
|
||||
it := db.NewIterator()
|
||||
defer it.Release()
|
||||
|
||||
var (
|
||||
count int64
|
||||
start = time.Now()
|
||||
logged = time.Now()
|
||||
|
||||
// Key-value store statistics
|
||||
total common.StorageSize
|
||||
headerSize common.StorageSize
|
||||
bodySize common.StorageSize
|
||||
receiptSize common.StorageSize
|
||||
tdSize common.StorageSize
|
||||
numHashPairing common.StorageSize
|
||||
hashNumPairing common.StorageSize
|
||||
trieSize common.StorageSize
|
||||
txlookupSize common.StorageSize
|
||||
preimageSize common.StorageSize
|
||||
bloomBitsSize common.StorageSize
|
||||
|
||||
// Ancient store statistics
|
||||
ancientHeaders common.StorageSize
|
||||
ancientBodies common.StorageSize
|
||||
ancientReceipts common.StorageSize
|
||||
ancientHashes common.StorageSize
|
||||
ancientTds common.StorageSize
|
||||
|
||||
// Les statistic
|
||||
ChtTrieNodes common.StorageSize
|
||||
BloomTrieNodes common.StorageSize
|
||||
)
|
||||
// Inspect key-value database first.
|
||||
for it.Next() {
|
||||
var (
|
||||
key = it.Key()
|
||||
size = common.StorageSize(len(key) + len(it.Value()))
|
||||
)
|
||||
total += size
|
||||
switch {
|
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
|
||||
tdSize += size
|
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
|
||||
numHashPairing += size
|
||||
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
|
||||
headerSize += size
|
||||
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
|
||||
hashNumPairing += size
|
||||
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
|
||||
bodySize += size
|
||||
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
|
||||
receiptSize += size
|
||||
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
|
||||
txlookupSize += size
|
||||
case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
|
||||
preimageSize += size
|
||||
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
|
||||
bloomBitsSize += size
|
||||
case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength:
|
||||
ChtTrieNodes += size
|
||||
case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength:
|
||||
BloomTrieNodes += size
|
||||
case len(key) == common.HashLength:
|
||||
trieSize += size
|
||||
}
|
||||
count += 1
|
||||
if count%1000 == 0 && time.Since(logged) > 8*time.Second {
|
||||
log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
logged = time.Now()
|
||||
}
|
||||
}
|
||||
// Inspect append-only file store then.
|
||||
ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds}
|
||||
for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} {
|
||||
if size, err := db.AncientSize(category); err == nil {
|
||||
*ancients[i] += common.StorageSize(size)
|
||||
total += common.StorageSize(size)
|
||||
}
|
||||
}
|
||||
// Display the database statistic.
|
||||
stats := [][]string{
|
||||
{"Key-Value store", "Headers", headerSize.String()},
|
||||
{"Key-Value store", "Bodies", bodySize.String()},
|
||||
{"Key-Value store", "Receipts", receiptSize.String()},
|
||||
{"Key-Value store", "Difficulties", tdSize.String()},
|
||||
{"Key-Value store", "Block number->hash", numHashPairing.String()},
|
||||
{"Key-Value store", "Block hash->number", hashNumPairing.String()},
|
||||
{"Key-Value store", "Transaction index", txlookupSize.String()},
|
||||
{"Key-Value store", "Bloombit index", bloomBitsSize.String()},
|
||||
{"Key-Value store", "Trie nodes", trieSize.String()},
|
||||
{"Key-Value store", "Trie preimages", preimageSize.String()},
|
||||
{"Ancient store", "Headers", ancientHeaders.String()},
|
||||
{"Ancient store", "Bodies", ancientBodies.String()},
|
||||
{"Ancient store", "Receipts", ancientReceipts.String()},
|
||||
{"Ancient store", "Difficulties", ancientTds.String()},
|
||||
{"Ancient store", "Block number->hash", ancientHashes.String()},
|
||||
{"Light client", "CHT trie nodes", ChtTrieNodes.String()},
|
||||
{"Light client", "Bloom trie nodes", BloomTrieNodes.String()},
|
||||
}
|
||||
table := tablewriter.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"Database", "Category", "Size"})
|
||||
table.SetFooter([]string{"", "Total", total.String()})
|
||||
table.AppendBulk(stats)
|
||||
table.Render()
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user