core/rawdb: simple legacy receipt converter (#24028)

* cmd,core: add simple legacy receipt converter

core/rawdb: use forEach in migrate

core/rawdb: batch reads in forEach

core/rawdb: make forEach anonymous fn

cmd/geth: check for legacy receipts on node startup

fix err msg

Co-authored-by: rjl493456442 <garyrong0905@gmail.com>

fix log

Co-authored-by: rjl493456442 <garyrong0905@gmail.com>

fix some review comments

add warning to cmd

drop isLegacy fn from migrateTable params

add test for windows rename

test replacing in windows case

* minor fix

* sanity check for tail-deletion

* add log before moving files around

* speed-up hack for mainnet

* fix mainnet check, use networkid instead

* check mainnet genesis

* review fixes

* resume previous migration attempt

* core/rawdb: lint fix

Co-authored-by: Martin Holst Swende <martin@swende.se>

cmd/geth/config.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/scwallet"
 	"github.com/ethereum/go-ethereum/accounts/usbwallet"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
@@ -161,7 +162,24 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) {
 		cfg.Eth.OverrideTerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTerminalTotalDifficulty.Name))
 	}
-	backend, _ := utils.RegisterEthService(stack, &cfg.Eth)
+	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
+	// Warn users to migrate if they have a legacy freezer format.
+	if eth != nil {
+		firstIdx := uint64(0)
+		// Hack to speed up check for mainnet because we know
+		// the first non-empty block.
+		ghash := rawdb.ReadCanonicalHash(eth.ChainDb(), 0)
+		if cfg.Eth.NetworkId == 1 && ghash == params.MainnetGenesisHash {
+			firstIdx = 46147
+		}
+		isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx)
+		if err != nil {
+			utils.Fatalf("Failed to check db for legacy receipts: %v", err)
+		}
+		if isLegacy {
+			log.Warn("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.")
+		}
+	}
 
 	// Configure GraphQL if requested
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {

cmd/geth/dbcmd.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
@@ -72,6 +73,7 @@ Remove blockchain and state databases`,
 			dbImportCmd,
 			dbExportCmd,
 			dbMetadataCmd,
+			dbMigrateFreezerCmd,
 		},
 	}
 	dbInspectCmd = cli.Command{
@@ -251,6 +253,23 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		},
 		Description: "Shows metadata about the chain status.",
 	}
+	dbMigrateFreezerCmd = cli.Command{
+		Action:    utils.MigrateFlags(freezerMigrate),
+		Name:      "freezer-migrate",
+		Usage:     "Migrate legacy parts of the freezer. (WARNING: may take a long time)",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.SepoliaFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+		},
+		Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those.
+WARNING: please back-up the receipt files in your ancients before running this command.`,
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -750,3 +769,88 @@ func showMetaData(ctx *cli.Context) error {
 	table.Render()
 	return nil
 }
+
+func freezerMigrate(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
+
+	// Check first block for legacy receipt format
+	numAncients, err := db.Ancients()
+	if err != nil {
+		return err
+	}
+	if numAncients < 1 {
+		log.Info("No receipts in freezer to migrate")
+		return nil
+	}
+
+	isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db, 0)
+	if err != nil {
+		return err
+	}
+	if !isFirstLegacy {
+		log.Info("No legacy receipts to migrate")
+		return nil
+	}
+
+	log.Info("Starting migration", "ancients", numAncients, "firstLegacy", firstIdx)
+	start := time.Now()
+	if err := db.MigrateTable("receipts", types.ConvertLegacyStoredReceipts); err != nil {
+		return err
+	}
+	if err := db.Close(); err != nil {
+		return err
+	}
+	log.Info("Migration finished", "duration", time.Since(start))
+
+	return nil
+}
+
+// dbHasLegacyReceipts checks freezer entries for legacy receipts. It stops at the first
+// non-empty receipt and checks its format. The index of this first non-empty element is
+// the second return parameter.
+func dbHasLegacyReceipts(db ethdb.Database, firstIdx uint64) (bool, uint64, error) {
+	// Check first block for legacy receipt format
+	numAncients, err := db.Ancients()
+	if err != nil {
+		return false, 0, err
+	}
+	if numAncients < 1 {
+		return false, 0, nil
+	}
+	if firstIdx >= numAncients {
+		return false, firstIdx, nil
+	}
+	var (
+		legacy       bool
+		blob         []byte
+		emptyRLPList = []byte{192}
+	)
+	// Find first block with non-empty receipt, only if
+	// the index is not already provided.
+	if firstIdx == 0 {
+		for i := uint64(0); i < numAncients; i++ {
+			blob, err = db.Ancient("receipts", i)
+			if err != nil {
+				return false, 0, err
+			}
+			if len(blob) == 0 {
+				continue
+			}
+			if !bytes.Equal(blob, emptyRLPList) {
+				firstIdx = i
+				break
+			}
+		}
+	}
+	// Is first non-empty receipt legacy?
+	first, err := db.Ancient("receipts", firstIdx)
+	if err != nil {
+		return false, 0, err
+	}
+	legacy, err = types.IsLegacyStoredReceipts(first)
+	return legacy, firstIdx, err
+}