core: lookup txs by block number instead of block hash (#19431)
* core: lookup txs by block number instead of block hash Transaction hashes now store a reference to their corresponding block number as opposed to their hash. In benchmarks this was shown to reduce storage by over 12 GB. The main limitation of this approach is that transactions on non-canonical blocks could never be looked up, however that is currently not supported. The database version has been upgraded to version 5 and the transaction lookup process is backwards-compatible with the prior two transaction lookup formats prexisting in the database instance. Tests have been added to ensure this. * core/rawdb: tiny review nit fixes
This commit is contained in:
Matthew Halpern
committed by
Péter Szilágyi
Péter Szilágyi
parent
7c91038bff
commit
937417527c
@ -22,69 +22,87 @@ import (
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// Tests that positional lookup metadata can be stored and retrieved.
|
||||
func TestLookupStorage(t *testing.T) {
|
||||
db := NewMemoryDatabase()
|
||||
|
||||
tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
|
||||
tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
|
||||
tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
|
||||
txs := []*types.Transaction{tx1, tx2, tx3}
|
||||
|
||||
block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil)
|
||||
|
||||
// Check that no transactions entries are in a pristine database
|
||||
for i, tx := range txs {
|
||||
if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
|
||||
t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn)
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
writeTxLookupEntries func(ethdb.Writer, *types.Block)
|
||||
}{
|
||||
{
|
||||
"DatabaseV6",
|
||||
func(db ethdb.Writer, block *types.Block) {
|
||||
WriteTxLookupEntries(db, block)
|
||||
},
|
||||
},
|
||||
{
|
||||
"DatabaseV4-V5",
|
||||
func(db ethdb.Writer, block *types.Block) {
|
||||
for _, tx := range block.Transactions() {
|
||||
db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes())
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"DatabaseV3",
|
||||
func(db ethdb.Writer, block *types.Block) {
|
||||
for index, tx := range block.Transactions() {
|
||||
entry := LegacyTxLookupEntry{
|
||||
BlockHash: block.Hash(),
|
||||
BlockIndex: block.NumberU64(),
|
||||
Index: uint64(index),
|
||||
}
|
||||
data, _ := rlp.EncodeToBytes(entry)
|
||||
db.Put(txLookupKey(tx.Hash()), data)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
// Insert all the transactions into the database, and verify contents
|
||||
WriteBlock(db, block)
|
||||
WriteTxLookupEntries(db, block)
|
||||
|
||||
for i, tx := range txs {
|
||||
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
|
||||
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
|
||||
} else {
|
||||
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
|
||||
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
db := NewMemoryDatabase()
|
||||
|
||||
tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
|
||||
tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
|
||||
tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
|
||||
txs := []*types.Transaction{tx1, tx2, tx3}
|
||||
|
||||
block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil)
|
||||
|
||||
// Check that no transactions entries are in a pristine database
|
||||
for i, tx := range txs {
|
||||
if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
|
||||
t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn)
|
||||
}
|
||||
}
|
||||
if tx.Hash() != txn.Hash() {
|
||||
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
|
||||
// Insert all the transactions into the database, and verify contents
|
||||
WriteCanonicalHash(db, block.Hash(), block.NumberU64())
|
||||
WriteBlock(db, block)
|
||||
tc.writeTxLookupEntries(db, block)
|
||||
|
||||
for i, tx := range txs {
|
||||
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
|
||||
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
|
||||
} else {
|
||||
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
|
||||
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
|
||||
}
|
||||
if tx.Hash() != txn.Hash() {
|
||||
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Delete the transactions and check purge
|
||||
for i, tx := range txs {
|
||||
DeleteTxLookupEntry(db, tx.Hash())
|
||||
if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
|
||||
t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
|
||||
}
|
||||
}
|
||||
// Insert legacy txlookup and verify the data retrieval
|
||||
for index, tx := range block.Transactions() {
|
||||
entry := LegacyTxLookupEntry{
|
||||
BlockHash: block.Hash(),
|
||||
BlockIndex: block.NumberU64(),
|
||||
Index: uint64(index),
|
||||
}
|
||||
data, _ := rlp.EncodeToBytes(entry)
|
||||
db.Put(txLookupKey(tx.Hash()), data)
|
||||
}
|
||||
for i, tx := range txs {
|
||||
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
|
||||
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
|
||||
} else {
|
||||
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
|
||||
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
|
||||
// Delete the transactions and check purge
|
||||
for i, tx := range txs {
|
||||
DeleteTxLookupEntry(db, tx.Hash())
|
||||
if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
|
||||
t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
|
||||
}
|
||||
}
|
||||
if tx.Hash() != txn.Hash() {
|
||||
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user