gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

core, trie: intermediate mempool between trie and database (#15857)

Browse Source 
This commit reduces database I/O by not writing every state trie to disk.
This commit is contained in:
Péter Szilágyi
2018-02-05 18:40:32 +02:00
committed by Felix Lange
parent 59336283c0
commit 55599ee95d
69 changed files with 1958 additions and 1164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/evm/runner.go
View File

@ -96,7 +96,9 @@ func runCmd(ctx *cli.Context) error {
}
if ctx.GlobalString(GenesisFlag.Name) != "" {
gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
_, statedb = gen.ToBlock()
db, _ := ethdb.NewMemDatabase()
genesis := gen.ToBlock(db)
statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
chainConfig = gen.Config
} else {
db, _ := ethdb.NewMemDatabase()

4
cmd/geth/chaincmd.go
View File

@ -202,7 +202,7 @@ func importChain(ctx *cli.Context) error {
if len(ctx.Args()) == 1 {
if err := utils.ImportChain(chain, ctx.Args().First()); err != nil {
utils.Fatalf("Import error: %v", err)
log.Error("Import error", "err", err)
}
} else {
for _, arg := range ctx.Args() {
@ -211,7 +211,7 @@ func importChain(ctx *cli.Context) error {
}
}
}
chain.Stop()
fmt.Printf("Import done in %v.\n\n", time.Since(start))
// Output pre-compaction stats mostly to see the import trashing

3
cmd/geth/main.go
View File

@ -85,10 +85,13 @@ var (
utils.FastSyncFlag,
utils.LightModeFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.LightServFlag,
utils.LightPeersFlag,
utils.LightKDFFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
utils.ListenPortFlag,
utils.MaxPeersFlag,

6
cmd/geth/usage.go
View File

@ -22,10 +22,11 @@ import (
"io"
"sort"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/internal/debug"
"gopkg.in/urfave/cli.v1"
"strings"
)
// AppHelpTemplate is the test template for the default, global app help topic.
@ -74,6 +75,7 @@ var AppHelpFlagGroups = []flagGroup{
utils.TestnetFlag,
utils.RinkebyFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.EthStatsURLFlag,
utils.IdentityFlag,
utils.LightServFlag,
@ -127,6 +129,8 @@ var AppHelpFlagGroups = []flagGroup{
Name: "PERFORMANCE TUNING",
Flags: []cli.Flag{
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheGCFlag,
utils.TrieCacheGenFlag,
},
},

26
cmd/utils/cmd.go
View File

@ -116,7 +116,6 @@ func ImportChain(chain *core.BlockChain, fn string) error {
return err
}
}
stream := rlp.NewStream(reader, 0)
// Run actual the import.
@ -150,25 +149,34 @@ func ImportChain(chain *core.BlockChain, fn string) error {
if checkInterrupt() {
return fmt.Errorf("interrupted")
}
if hasAllBlocks(chain, blocks[:i]) {
missing := missingBlocks(chain, blocks[:i])
if len(missing) == 0 {
log.Info("Skipping batch as all blocks present", "batch", batch, "first", blocks[0].Hash(), "last", blocks[i-1].Hash())
continue
}
if _, err := chain.InsertChain(blocks[:i]); err != nil {
if _, err := chain.InsertChain(missing); err != nil {
return fmt.Errorf("invalid block %d: %v", n, err)
}
}
return nil
}
func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool {
for _, b := range bs {
if !chain.HasBlock(b.Hash(), b.NumberU64()) {
return false
func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block {
head := chain.CurrentBlock()
for i, block := range blocks {
// If we're behind the chain head, only check block, state is available at head
if head.NumberU64() > block.NumberU64() {
if !chain.HasBlock(block.Hash(), block.NumberU64()) {
return blocks[i:]
}
continue
}
// If we're above the chain head, state availability is a must
if !chain.HasBlockAndState(block.Hash(), block.NumberU64()) {
return blocks[i:]
}
}
return true
return nil
}
func ExportChain(blockchain *core.BlockChain, fn string) error {

47
cmd/utils/flags.go
View File

@ -170,7 +170,11 @@ var (
Usage: `Blockchain sync mode ("fast", "full", or "light")`,
Value: &defaultSyncMode,
}
GCModeFlag = cli.StringFlag{
Name: "gcmode",
Usage: `Blockchain garbage collection mode ("full", "archive")`,
Value: "full",
}
LightServFlag = cli.IntFlag{
Name: "lightserv",
Usage: "Maximum percentage of time allowed for serving LES requests (0-90)",
@ -293,8 +297,18 @@ var (
// Performance tuning settings
CacheFlag = cli.IntFlag{
Name: "cache",
Usage: "Megabytes of memory allocated to internal caching (min 16MB / database forced)",
Value: 128,
Usage: "Megabytes of memory allocated to internal caching",
Value: 1024,
}
CacheDatabaseFlag = cli.IntFlag{
Name: "cache.database",
Usage: "Percentage of cache memory allowance to use for database io",
Value: 75,
}
CacheGCFlag = cli.IntFlag{
Name: "cache.gc",
Usage: "Percentage of cache memory allowance to use for trie pruning",
Value: 25,
}
TrieCacheGenFlag = cli.IntFlag{
Name: "trie-cache-gens",
@ -1021,11 +1035,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
}
if ctx.GlobalIsSet(CacheFlag.Name) {
cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name)
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
}
cfg.DatabaseHandles = makeDatabaseHandles()
if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
}
@ -1157,7 +1179,7 @@ func SetupNetwork(ctx *cli.Context) {
// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
var (
cache = ctx.GlobalInt(CacheFlag.Name)
cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
handles = makeDatabaseHandles()
)
name := "chaindata"
@ -1209,8 +1231,19 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
})
}
}
if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cache := &core.CacheConfig{
Disabled: ctx.GlobalString(GCModeFlag.Name) == "archive",
TrieNodeLimit: eth.DefaultConfig.TrieCache,
TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
}
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
}
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
chain, err = core.NewBlockChain(chainDb, config, engine, vmcfg)
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg)
if err != nil {
Fatalf("Can't create BlockChain: %v", err)
}