gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

all: add read-only option to database (#22407)

Browse Source 
* all: add read-only option to database

* all: fixes tests

* cmd/geth: migrate flags

* cmd/geth: fix the compact

* cmd/geth: fix the format

* cmd/geth: fix log

* cmd: add chain-readonly

* core: add readonly notion to freezer

* core/rawdb: add log

* core/rawdb: fix freezer close

* cmd: fix

* cmd, core: construct db

* core: update tests
This commit is contained in:
gary rong
2021-03-23 02:06:30 +08:00
committed by GitHub
parent aab35600bc
commit 0c70b83e00
26 changed files with 267 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
cmd/geth/chaincmd.go
View File

@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
@ -191,7 +192,7 @@ func initGenesis(ctx *cli.Context) error {
defer stack.Close()
for _, name := range []string{"chaindata", "lightchaindata"} {
chaindb, err := stack.OpenDatabase(name, 0, 0, "")
chaindb, err := stack.OpenDatabase(name, 0, 0, "", false)
if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}
@ -229,7 +230,7 @@ func importChain(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, db := utils.MakeChain(ctx, stack, false)
chain, db := utils.MakeChain(ctx, stack)
defer db.Close()
// Start periodically gathering memory profiles
@ -304,7 +305,7 @@ func exportChain(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, _ := utils.MakeChain(ctx, stack, true)
chain, _ := utils.MakeChain(ctx, stack)
start := time.Now()
var err error
@ -340,7 +341,7 @@ func importPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, false)
start := time.Now()
if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
@ -359,7 +360,7 @@ func exportPreimages(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, true)
start := time.Now()
if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
@ -373,21 +374,27 @@ func dump(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
db := utils.MakeChainDatabase(ctx, stack, true)
for _, arg := range ctx.Args() {
var block *types.Block
var header *types.Header
if hashish(arg) {
block = chain.GetBlockByHash(common.HexToHash(arg))
hash := common.HexToHash(arg)
number := rawdb.ReadHeaderNumber(db, hash)
if number != nil {
header = rawdb.ReadHeader(db, hash, *number)
}
} else {
num, _ := strconv.Atoi(arg)
block = chain.GetBlockByNumber(uint64(num))
number, _ := strconv.Atoi(arg)
hash := rawdb.ReadCanonicalHash(db, uint64(number))
if hash != (common.Hash{}) {
header = rawdb.ReadHeader(db, hash, uint64(number))
}
}
if block == nil {
if header == nil {
fmt.Println("{}")
utils.Fatalf("block not found")
} else {
state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil)
state, err := state.New(header.Root, state.NewDatabase(db), nil)
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}

2
cmd/geth/dao_test.go
View File

@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
}
// Retrieve the DAO config flag from the database
path := filepath.Join(datadir, "geth", "chaindata")
db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "")
db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false)
if err != nil {
t.Fatalf("test %d: failed to open test database: %v", test, err)
}

155
cmd/geth/dbcmd.go
View File

@ -28,9 +28,7 @@ import (
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/log"
"github.com/syndtr/goleveldb/leveldb/opt"
"gopkg.in/urfave/cli.v1"
)
@ -65,43 +63,98 @@ Remove blockchain and state databases`,
Action: utils.MigrateFlags(inspect),
Name: "inspect",
ArgsUsage: "<prefix> <start>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Usage: "Inspect the storage size for each type of data in the database",
Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
}
dbStatCmd = cli.Command{
Action: dbStats,
Action: utils.MigrateFlags(dbStats),
Name: "stats",
Usage: "Print leveldb statistics",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
}
dbCompactCmd = cli.Command{
Action: dbCompact,
Action: utils.MigrateFlags(dbCompact),
Name: "compact",
Usage: "Compact leveldb database. WARNING: May take a very long time",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
},
Description: `This command performs a database compaction.
WARNING: This operation may take a very long time to finish, and may cause database
corruption if it is aborted during execution'!`,
}
dbGetCmd = cli.Command{
Action: dbGet,
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Action: utils.MigrateFlags(dbGet),
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: "This command looks up the specified database key from the database.",
}
dbDeleteCmd = cli.Command{
Action: dbDelete,
Action: utils.MigrateFlags(dbDelete),
Name: "delete",
Usage: "Delete a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: `This command deletes the specified database key from the database.
WARNING: This is a low-level operation which may cause database corruption!`,
}
dbPutCmd = cli.Command{
Action: dbPut,
Action: utils.MigrateFlags(dbPut),
Name: "put",
Usage: "Set the value of a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key> <hex-encoded value>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.SyncModeFlag,
utils.MainnetFlag,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV3Flag,
},
Description: `This command sets a given database key to the given value.
WARNING: This is a low-level operation which may cause database corruption!`,
}
@ -192,10 +245,10 @@ func inspect(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
_, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
return rawdb.InspectDatabase(chainDb, prefix, start)
return rawdb.InspectDatabase(db, prefix, start)
}
func showLeveldbStats(db ethdb.Stater) {
@ -214,48 +267,32 @@ func showLeveldbStats(db ethdb.Stater) {
func dbStats(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.ReadOnly = true
})
if err != nil {
return err
}
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
showLeveldbStats(db)
err = db.Close()
if err != nil {
log.Info("Close err", "error", err)
}
return nil
}
func dbCompact(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles()
options.BlockCacheCapacity = cache / 2 * opt.MiB
options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
})
if err != nil {
db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()
log.Info("Stats before compaction")
showLeveldbStats(db)
log.Info("Triggering compaction")
if err := db.Compact(nil, nil); err != nil {
log.Info("Compact err", "error", err)
return err
}
log.Info("Stats after compaction")
showLeveldbStats(db)
log.Info("Triggering compaction")
err = db.Compact(nil, nil)
if err != nil {
log.Info("Compact err", "error", err)
}
showLeveldbStats(db)
log.Info("Closing db")
err = db.Close()
if err != nil {
log.Info("Close err", "error", err)
}
log.Info("Exiting")
return err
return nil
}
// dbGet shows the value of a given database key
@ -265,14 +302,10 @@ func dbGet(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
options.ReadOnly = true
})
if err != nil {
return err
}
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
key, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the key", "error", err)
@ -283,7 +316,7 @@ func dbGet(ctx *cli.Context) error {
log.Info("Get operation failed", "error", err)
return err
}
fmt.Printf("key %#x:\n\t%#x\n", key, data)
fmt.Printf("key %#x: %#x\n", key, data)
return nil
}
@ -294,13 +327,19 @@ func dbDelete(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()
key, err := hexutil.Decode(ctx.Args().Get(0))
if err != nil {
log.Info("Could not decode the key", "error", err)
return err
}
data, err := db.Get(key)
if err == nil {
fmt.Printf("Previous value: %#x\n", data)
}
if err = db.Delete(key); err != nil {
log.Info("Delete operation returned an error", "error", err)
return err
@ -315,8 +354,10 @@ func dbPut(ctx *cli.Context) error {
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()
var (
key []byte
value []byte
@ -335,7 +376,7 @@ func dbPut(ctx *cli.Context) error {
}
data, err = db.Get(key)
if err == nil {
fmt.Printf("Previous value:\n%#x\n", data)
fmt.Printf("Previous value: %#x\n", data)
}
return db.Put(key, value)
}

57
cmd/geth/snapshot.go
View File

@ -152,10 +152,8 @@ func pruneState(ctx *cli.Context) error {
stack, config := makeConfigNode(ctx)
defer stack.Close()
chain, chaindb := utils.MakeChain(ctx, stack, true)
defer chaindb.Close()
pruner, err := pruner.NewPruner(chaindb, chain.CurrentBlock().Header(), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name))
chaindb := utils.MakeChainDatabase(ctx, stack, false)
pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name))
if err != nil {
log.Error("Failed to open snapshot tree", "error", err)
return err
@ -183,10 +181,13 @@ func verifyState(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, chaindb := utils.MakeChain(ctx, stack, true)
defer chaindb.Close()
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, chain.CurrentBlock().Root(), false, false, false)
chaindb := utils.MakeChainDatabase(ctx, stack, true)
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
log.Error("Failed to load head block")
return errors.New("no head block")
}
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false)
if err != nil {
log.Error("Failed to open snapshot tree", "error", err)
return err
@ -195,7 +196,7 @@ func verifyState(ctx *cli.Context) error {
log.Error("Too many arguments given")
return errors.New("too many arguments")
}
var root = chain.CurrentBlock().Root()
var root = headBlock.Root()
if ctx.NArg() == 1 {
root, err = parseRoot(ctx.Args()[0])
if err != nil {
@ -218,19 +219,16 @@ func traverseState(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, chaindb := utils.MakeChain(ctx, stack, true)
defer chaindb.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true)
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
log.Error("Failed to load head block")
return errors.New("no head block")
}
if ctx.NArg() > 1 {
log.Error("Too many arguments given")
return errors.New("too many arguments")
}
// Use the HEAD root as the default
head := chain.CurrentBlock()
if head == nil {
log.Error("Head block is missing")
return errors.New("head block is missing")
}
var (
root common.Hash
err error
@ -243,8 +241,8 @@ func traverseState(ctx *cli.Context) error {
}
log.Info("Start traversing the state", "root", root)
} else {
root = head.Root()
log.Info("Start traversing the state", "root", root, "number", head.NumberU64())
root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(root, triedb)
@ -311,19 +309,16 @@ func traverseRawState(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, chaindb := utils.MakeChain(ctx, stack, true)
defer chaindb.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true)
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
log.Error("Failed to load head block")
return errors.New("no head block")
}
if ctx.NArg() > 1 {
log.Error("Too many arguments given")
return errors.New("too many arguments")
}
// Use the HEAD root as the default
head := chain.CurrentBlock()
if head == nil {
log.Error("Head block is missing")
return errors.New("head block is missing")
}
var (
root common.Hash
err error
@ -336,8 +331,8 @@ func traverseRawState(ctx *cli.Context) error {
}
log.Info("Start traversing the state", "root", root)
} else {
root = head.Root()
log.Info("Start traversing the state", "root", root, "number", head.NumberU64())
root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(root, triedb)