cmd/geth: implement data import and export (#22931)
This PR offers two more database sub commands for exporting and importing data. Two exporters are implemented: preimage and snapshot data respectively. The import command is generic, it can take any data export and import into leveldb. The data format has a 'magic' for disambiguation, and a version field for future compatibility.
This commit is contained in:
rjl493456442
GitHub
@ -140,7 +140,9 @@ be gzipped.`,
|
||||
},
|
||||
Category: "BLOCKCHAIN COMMANDS",
|
||||
Description: `
|
||||
The import-preimages command imports hash preimages from an RLP encoded stream.`,
|
||||
The import-preimages command imports hash preimages from an RLP encoded stream.
|
||||
It's deprecated, please use "geth db import" instead.
|
||||
`,
|
||||
}
|
||||
exportPreimagesCommand = cli.Command{
|
||||
Action: utils.MigrateFlags(exportPreimages),
|
||||
@ -154,7 +156,9 @@ be gzipped.`,
|
||||
},
|
||||
Category: "BLOCKCHAIN COMMANDS",
|
||||
Description: `
|
||||
The export-preimages command export hash preimages to an RLP encoded stream`,
|
||||
The export-preimages command exports hash preimages to an RLP encoded stream.
|
||||
It's deprecated, please use "geth db export" instead.
|
||||
`,
|
||||
}
|
||||
dumpCommand = cli.Command{
|
||||
Action: utils.MigrateFlags(dump),
|
||||
@ -368,7 +372,6 @@ func exportPreimages(ctx *cli.Context) error {
|
||||
if len(ctx.Args()) < 1 {
|
||||
utils.Fatalf("This command requires an argument.")
|
||||
}
|
||||
|
||||
stack, _ := makeConfigNode(ctx)
|
||||
defer stack.Close()
|
||||
|
||||
|
||||
@ -17,12 +17,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
@ -63,6 +67,8 @@ Remove blockchain and state databases`,
|
||||
dbPutCmd,
|
||||
dbGetSlotsCmd,
|
||||
dbDumpFreezerIndex,
|
||||
dbImportCmd,
|
||||
dbExportCmd,
|
||||
},
|
||||
}
|
||||
dbInspectCmd = cli.Command{
|
||||
@ -188,6 +194,36 @@ WARNING: This is a low-level operation which may cause database corruption!`,
|
||||
},
|
||||
Description: "This command displays information about the freezer index.",
|
||||
}
|
||||
dbImportCmd = cli.Command{
|
||||
Action: utils.MigrateFlags(importLDBdata),
|
||||
Name: "import",
|
||||
Usage: "Imports leveldb-data from an exported RLP dump.",
|
||||
ArgsUsage: "<dumpfile> <start (optional)",
|
||||
Flags: []cli.Flag{
|
||||
utils.DataDirFlag,
|
||||
utils.SyncModeFlag,
|
||||
utils.MainnetFlag,
|
||||
utils.RopstenFlag,
|
||||
utils.RinkebyFlag,
|
||||
utils.GoerliFlag,
|
||||
},
|
||||
Description: "The import command imports the specific chain data from an RLP encoded stream.",
|
||||
}
|
||||
dbExportCmd = cli.Command{
|
||||
Action: utils.MigrateFlags(exportChaindata),
|
||||
Name: "export",
|
||||
Usage: "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used.",
|
||||
ArgsUsage: "<type> <dumpfile>",
|
||||
Flags: []cli.Flag{
|
||||
utils.DataDirFlag,
|
||||
utils.SyncModeFlag,
|
||||
utils.MainnetFlag,
|
||||
utils.RopstenFlag,
|
||||
utils.RinkebyFlag,
|
||||
utils.GoerliFlag,
|
||||
},
|
||||
Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
|
||||
}
|
||||
)
|
||||
|
||||
func removeDB(ctx *cli.Context) error {
|
||||
@ -510,3 +546,133 @@ func parseHexOrString(str string) ([]byte, error) {
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
func importLDBdata(ctx *cli.Context) error {
|
||||
start := 0
|
||||
switch ctx.NArg() {
|
||||
case 1:
|
||||
break
|
||||
case 2:
|
||||
s, err := strconv.Atoi(ctx.Args().Get(1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("second arg must be an integer: %v", err)
|
||||
}
|
||||
start = s
|
||||
default:
|
||||
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
|
||||
}
|
||||
var (
|
||||
fName = ctx.Args().Get(0)
|
||||
stack, _ = makeConfigNode(ctx)
|
||||
interrupt = make(chan os.Signal, 1)
|
||||
stop = make(chan struct{})
|
||||
)
|
||||
defer stack.Close()
|
||||
signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
|
||||
defer signal.Stop(interrupt)
|
||||
defer close(interrupt)
|
||||
go func() {
|
||||
if _, ok := <-interrupt; ok {
|
||||
log.Info("Interrupted during ldb import, stopping at next batch")
|
||||
}
|
||||
close(stop)
|
||||
}()
|
||||
db := utils.MakeChainDatabase(ctx, stack, false)
|
||||
return utils.ImportLDBData(db, fName, int64(start), stop)
|
||||
}
|
||||
|
||||
type preimageIterator struct {
|
||||
iter ethdb.Iterator
|
||||
}
|
||||
|
||||
func (iter *preimageIterator) Next() (byte, []byte, []byte, bool) {
|
||||
for iter.iter.Next() {
|
||||
key := iter.iter.Key()
|
||||
if bytes.HasPrefix(key, rawdb.PreimagePrefix) && len(key) == (len(rawdb.PreimagePrefix)+common.HashLength) {
|
||||
return utils.OpBatchAdd, key, iter.iter.Value(), true
|
||||
}
|
||||
}
|
||||
return 0, nil, nil, false
|
||||
}
|
||||
|
||||
func (iter *preimageIterator) Release() {
|
||||
iter.iter.Release()
|
||||
}
|
||||
|
||||
type snapshotIterator struct {
|
||||
init bool
|
||||
account ethdb.Iterator
|
||||
storage ethdb.Iterator
|
||||
}
|
||||
|
||||
func (iter *snapshotIterator) Next() (byte, []byte, []byte, bool) {
|
||||
if !iter.init {
|
||||
iter.init = true
|
||||
return utils.OpBatchDel, rawdb.SnapshotRootKey, nil, true
|
||||
}
|
||||
for iter.account.Next() {
|
||||
key := iter.account.Key()
|
||||
if bytes.HasPrefix(key, rawdb.SnapshotAccountPrefix) && len(key) == (len(rawdb.SnapshotAccountPrefix)+common.HashLength) {
|
||||
return utils.OpBatchAdd, key, iter.account.Value(), true
|
||||
}
|
||||
}
|
||||
for iter.storage.Next() {
|
||||
key := iter.storage.Key()
|
||||
if bytes.HasPrefix(key, rawdb.SnapshotStoragePrefix) && len(key) == (len(rawdb.SnapshotStoragePrefix)+2*common.HashLength) {
|
||||
return utils.OpBatchAdd, key, iter.storage.Value(), true
|
||||
}
|
||||
}
|
||||
return 0, nil, nil, false
|
||||
}
|
||||
|
||||
func (iter *snapshotIterator) Release() {
|
||||
iter.account.Release()
|
||||
iter.storage.Release()
|
||||
}
|
||||
|
||||
// chainExporters defines the export scheme for all exportable chain data.
|
||||
var chainExporters = map[string]func(db ethdb.Database) utils.ChainDataIterator{
|
||||
"preimage": func(db ethdb.Database) utils.ChainDataIterator {
|
||||
iter := db.NewIterator(rawdb.PreimagePrefix, nil)
|
||||
return &preimageIterator{iter: iter}
|
||||
},
|
||||
"snapshot": func(db ethdb.Database) utils.ChainDataIterator {
|
||||
account := db.NewIterator(rawdb.SnapshotAccountPrefix, nil)
|
||||
storage := db.NewIterator(rawdb.SnapshotStoragePrefix, nil)
|
||||
return &snapshotIterator{account: account, storage: storage}
|
||||
},
|
||||
}
|
||||
|
||||
func exportChaindata(ctx *cli.Context) error {
|
||||
if ctx.NArg() < 2 {
|
||||
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
|
||||
}
|
||||
// Parse the required chain data type, make sure it's supported.
|
||||
kind := ctx.Args().Get(0)
|
||||
kind = strings.ToLower(strings.Trim(kind, " "))
|
||||
exporter, ok := chainExporters[kind]
|
||||
if !ok {
|
||||
var kinds []string
|
||||
for kind := range chainExporters {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
return fmt.Errorf("invalid data type %s, supported types: %s", kind, strings.Join(kinds, ", "))
|
||||
}
|
||||
var (
|
||||
stack, _ = makeConfigNode(ctx)
|
||||
interrupt = make(chan os.Signal, 1)
|
||||
stop = make(chan struct{})
|
||||
)
|
||||
defer stack.Close()
|
||||
signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM)
|
||||
defer signal.Stop(interrupt)
|
||||
defer close(interrupt)
|
||||
go func() {
|
||||
if _, ok := <-interrupt; ok {
|
||||
log.Info("Interrupted during db export, stopping at next batch")
|
||||
}
|
||||
close(stop)
|
||||
}()
|
||||
db := utils.MakeChainDatabase(ctx, stack, true)
|
||||
return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user