gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

all: clean up and proerly abstract database access

Browse Source
This commit is contained in:
Péter Szilágyi
2018-09-24 15:57:49 +03:00
parent 15eee47ebf
commit 054412e335
94 changed files with 1573 additions and 1381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
trie/database.go
View File

@ -55,20 +55,11 @@ var secureKeyPrefix = []byte("secure-key-")
// secureKeyLength is the length of the above prefix + 32byte hash.
const secureKeyLength = 11 + 32
// DatabaseReader wraps the Get and Has method of a backing store for the trie.
type DatabaseReader interface {
// Get retrieves the value associated with key from the database.
Get(key []byte) (value []byte, err error)
// Has retrieves whether a key is present in the database.
Has(key []byte) (bool, error)
}
// Database is an intermediate write layer between the trie data structures and
// the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder.
type Database struct {
diskdb ethdb.Database // Persistent storage for matured trie nodes
diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes
cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes
@ -271,14 +262,14 @@ func expandNode(hash hashNode, n node, cachegen uint16) node {
// NewDatabase creates a new trie database to store ephemeral trie content before
// its written out to disk or garbage collected. No read cache is created, so all
// data retrievals will hit the underlying disk database.
func NewDatabase(diskdb ethdb.Database) *Database {
func NewDatabase(diskdb ethdb.KeyValueStore) *Database {
return NewDatabaseWithCache(diskdb, 0)
}
// NewDatabaseWithCache creates a new trie database to store ephemeral trie content
// before its written out to disk or garbage collected. It also acts as a read cache
// for nodes loaded from disk.
func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database {
func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database {
var cleans *bigcache.BigCache
if cache > 0 {
cleans, _ = bigcache.NewBigCache(bigcache.Config{
@ -298,7 +289,7 @@ func NewDatabaseWithCache(diskdb ethdb.Database, cache int) *Database {
}
// DiskDB retrieves the persistent storage backing the trie database.
func (db *Database) DiskDB() DatabaseReader {
func (db *Database) DiskDB() ethdb.Reader {
return db.diskdb
}

17
trie/iterator_test.go
View File

@ -23,7 +23,7 @@ import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
func TestIterator(t *testing.T) {
@ -120,11 +120,14 @@ func TestNodeIteratorCoverage(t *testing.T) {
}
}
}
for _, key := range db.diskdb.(*ethdb.MemDatabase).Keys() {
it := db.diskdb.NewIterator()
for it.Next() {
key := it.Key()
if _, ok := hashes[common.BytesToHash(key)]; !ok {
t.Errorf("state entry not reported %x", key)
}
}
it.Release()
}
type kvs struct{ k, v string }
@ -289,7 +292,7 @@ func TestIteratorContinueAfterErrorDisk(t *testing.T) { testIteratorContinueA
func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) }
func testIteratorContinueAfterError(t *testing.T, memonly bool) {
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
tr, _ := New(common.Hash{}, triedb)
@ -309,7 +312,11 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
if memonly {
memKeys = triedb.Nodes()
} else {
diskKeys = diskdb.Keys()
it := diskdb.NewIterator()
for it.Next() {
diskKeys = append(diskKeys, it.Key())
}
it.Release()
}
for i := 0; i < 20; i++ {
// Create trie that will load all nodes from DB.
@ -376,7 +383,7 @@ func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) {
func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
// Commit test trie to db, then remove the node containing "bars".
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
ctr, _ := New(common.Hash{}, triedb)

6
trie/proof.go
View File

@ -34,7 +34,7 @@ import (
// If the trie does not contain a value for key, the returned proof contains all
// nodes of the longest existing prefix of the key (at least the root node), ending
// with the node that proves the absence of the key.
func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error {
// Collect all nodes on the path to key.
key = keybytesToHex(key)
var nodes []node
@ -97,14 +97,14 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
// If the trie does not contain a value for key, the returned proof contains all
// nodes of the longest existing prefix of the key (at least the root node), ending
// with the node that proves the absence of the key.
func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Putter) error {
func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.Writer) error {
return t.trie.Prove(key, fromLevel, proofDb)
}
// VerifyProof checks merkle proofs. The given proof must contain the value for
// key in a trie with the given root hash. VerifyProof returns an error if the
// proof contains invalid trie nodes or the wrong value.
func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, nodes int, err error) {
func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.Reader) (value []byte, nodes int, err error) {
key = keybytesToHex(key)
wantHash := rootHash
for i := 0; ; i++ {

31
trie/proof_test.go
View File

@ -25,7 +25,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
func init() {
@ -34,18 +34,18 @@ func init() {
// makeProvers creates Merkle trie provers based on different implementations to
// test all variations.
func makeProvers(trie *Trie) []func(key []byte) *ethdb.MemDatabase {
var provers []func(key []byte) *ethdb.MemDatabase
func makeProvers(trie *Trie) []func(key []byte) *memorydb.MemoryDatabase {
var provers []func(key []byte) *memorydb.MemoryDatabase
// Create a direct trie based Merkle prover
provers = append(provers, func(key []byte) *ethdb.MemDatabase {
proof := ethdb.NewMemDatabase()
provers = append(provers, func(key []byte) *memorydb.MemoryDatabase {
proof := memorydb.New()
trie.Prove(key, 0, proof)
return proof
})
// Create a leaf iterator based Merkle prover
provers = append(provers, func(key []byte) *ethdb.MemDatabase {
proof := ethdb.NewMemDatabase()
provers = append(provers, func(key []byte) *memorydb.MemoryDatabase {
proof := memorydb.New()
if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) {
for _, p := range it.Prove() {
proof.Put(crypto.Keccak256(p), p)
@ -106,9 +106,14 @@ func TestBadProof(t *testing.T) {
if proof == nil {
t.Fatalf("prover %d: nil proof", i)
}
key := proof.Keys()[mrand.Intn(proof.Len())]
it := proof.NewIterator()
for i, d := 0, mrand.Intn(proof.Len()); i <= d; i++ {
it.Next()
}
key := it.Key()
val, _ := proof.Get(key)
proof.Delete(key)
it.Release()
mutateByte(val)
proof.Put(crypto.Keccak256(val), val)
@ -127,7 +132,7 @@ func TestMissingKeyProof(t *testing.T) {
updateString(trie, "k", "v")
for i, key := range []string{"a", "j", "l", "z"} {
proof := ethdb.NewMemDatabase()
proof := memorydb.New()
trie.Prove([]byte(key), 0, proof)
if proof.Len() != 1 {
@ -164,8 +169,8 @@ func BenchmarkProve(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
kv := vals[keys[i%len(keys)]]
proofs := ethdb.NewMemDatabase()
if trie.Prove(kv.k, 0, proofs); len(proofs.Keys()) == 0 {
proofs := memorydb.New()
if trie.Prove(kv.k, 0, proofs); proofs.Len() == 0 {
b.Fatalf("zero length proof for %x", kv.k)
}
}
@ -175,10 +180,10 @@ func BenchmarkVerifyProof(b *testing.B) {
trie, vals := randomTrie(100)
root := trie.Hash()
var keys []string
var proofs []*ethdb.MemDatabase
var proofs []*memorydb.MemoryDatabase
for k := range vals {
keys = append(keys, k)
proof := ethdb.NewMemDatabase()
proof := memorydb.New()
trie.Prove([]byte(k), 0, proof)
proofs = append(proofs, proof)
}

6
trie/secure_trie_test.go
View File

@ -24,18 +24,18 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
func newEmptySecure() *SecureTrie {
trie, _ := NewSecure(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()), 0)
trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New()), 0)
return trie
}
// makeTestSecureTrie creates a large enough secure trie for testing.
func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(ethdb.NewMemDatabase())
triedb := NewDatabase(memorydb.New())
trie, _ := NewSecure(common.Hash{}, triedb, 0)

6
trie/sync.go
View File

@ -72,14 +72,14 @@ func newSyncMemBatch() *syncMemBatch {
// unknown trie hashes to retrieve, accepts node data associated with said hashes
// and reconstructs the trie step by step until all is done.
type Sync struct {
database DatabaseReader // Persistent database to check for existing entries
database ethdb.Reader // Persistent database to check for existing entries
membatch *syncMemBatch // Memory buffer to avoid frequent database writes
requests map[common.Hash]*request // Pending requests pertaining to a key hash
queue *prque.Prque // Priority queue with the pending requests
}
// NewSync creates a new trie data download scheduler.
func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) *Sync {
func NewSync(root common.Hash, database ethdb.Reader, callback LeafCallback) *Sync {
ts := &Sync{
database: database,
membatch: newSyncMemBatch(),
@ -213,7 +213,7 @@ func (s *Sync) Process(results []SyncResult) (bool, int, error) {
// Commit flushes the data stored in the internal membatch out to persistent
// storage, returning the number of items written and any occurred error.
func (s *Sync) Commit(dbw ethdb.Putter) (int, error) {
func (s *Sync) Commit(dbw ethdb.Writer) (int, error) {
// Dump the membatch into a database dbw
for i, key := range s.membatch.order {
if err := dbw.Put(key[:], s.membatch.batch[key]); err != nil {

22
trie/sync_test.go
View File

@ -21,13 +21,13 @@ import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
// makeTestTrie create a sample test trie to test node-wise reconstruction.
func makeTestTrie() (*Database, *Trie, map[string][]byte) {
// Create an empty trie
triedb := NewDatabase(ethdb.NewMemDatabase())
triedb := NewDatabase(memorydb.New())
trie, _ := New(common.Hash{}, triedb)
// Fill it with some arbitrary data
@ -88,13 +88,13 @@ func checkTrieConsistency(db *Database, root common.Hash) error {
// Tests that an empty trie is not scheduled for syncing.
func TestEmptySync(t *testing.T) {
dbA := NewDatabase(ethdb.NewMemDatabase())
dbB := NewDatabase(ethdb.NewMemDatabase())
dbA := NewDatabase(memorydb.New())
dbB := NewDatabase(memorydb.New())
emptyA, _ := New(common.Hash{}, dbA)
emptyB, _ := New(emptyRoot, dbB)
for i, trie := range []*Trie{emptyA, emptyB} {
if req := NewSync(trie.Hash(), ethdb.NewMemDatabase(), nil).Missing(1); len(req) != 0 {
if req := NewSync(trie.Hash(), memorydb.New(), nil).Missing(1); len(req) != 0 {
t.Errorf("test %d: content requested for empty trie: %v", i, req)
}
}
@ -110,7 +110,7 @@ func testIterativeSync(t *testing.T, batch int) {
srcDb, srcTrie, srcData := makeTestTrie()
// Create a destination trie and sync with the scheduler
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil)
@ -143,7 +143,7 @@ func TestIterativeDelayedSync(t *testing.T) {
srcDb, srcTrie, srcData := makeTestTrie()
// Create a destination trie and sync with the scheduler
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil)
@ -181,7 +181,7 @@ func testIterativeRandomSync(t *testing.T, batch int) {
srcDb, srcTrie, srcData := makeTestTrie()
// Create a destination trie and sync with the scheduler
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil)
@ -222,7 +222,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
srcDb, srcTrie, srcData := makeTestTrie()
// Create a destination trie and sync with the scheduler
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil)
@ -269,7 +269,7 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
srcDb, srcTrie, srcData := makeTestTrie()
// Create a destination trie and sync with the scheduler
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil)
@ -309,7 +309,7 @@ func TestIncompleteSync(t *testing.T) {
srcDb, srcTrie, _ := makeTestTrie()
// Create a destination trie and sync with the scheduler
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
sched := NewSync(srcTrie.Hash(), diskdb, nil)

20
trie/trie_test.go
View File

@ -33,6 +33,8 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/rlp"
)
@ -43,7 +45,7 @@ func init() {
// Used for testing
func newEmpty() *Trie {
trie, _ := New(common.Hash{}, NewDatabase(ethdb.NewMemDatabase()))
trie, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
return trie
}
@ -67,7 +69,7 @@ func TestNull(t *testing.T) {
}
func TestMissingRoot(t *testing.T) {
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(ethdb.NewMemDatabase()))
trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New()))
if trie != nil {
t.Error("New returned non-nil trie for invalid root")
}
@ -80,7 +82,7 @@ func TestMissingNodeDisk(t *testing.T) { testMissingNode(t, false) }
func TestMissingNodeMemonly(t *testing.T) { testMissingNode(t, true) }
func testMissingNode(t *testing.T, memonly bool) {
diskdb := ethdb.NewMemDatabase()
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
trie, _ := New(common.Hash{}, triedb)
@ -317,13 +319,13 @@ func TestLargeValue(t *testing.T) {
}
type countingDB struct {
ethdb.Database
ethdb.KeyValueStore
gets map[string]int
}
func (db *countingDB) Get(key []byte) ([]byte, error) {
db.gets[string(key)]++
return db.Database.Get(key)
return db.KeyValueStore.Get(key)
}
// TestCacheUnload checks that decoded nodes are unloaded after a
@ -342,7 +344,7 @@ func TestCacheUnload(t *testing.T) {
// Commit the trie repeatedly and access key1.
// The branch containing it is loaded from DB exactly two times:
// in the 0th and 6th iteration.
diskdb := &countingDB{Database: trie.db.diskdb, gets: make(map[string]int)}
diskdb := &countingDB{KeyValueStore: trie.db.diskdb, gets: make(map[string]int)}
triedb := NewDatabase(diskdb)
trie, _ = New(root, triedb)
trie.SetCacheLimit(5)
@ -412,7 +414,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
}
func runRandTest(rt randTest) bool {
triedb := NewDatabase(ethdb.NewMemDatabase())
triedb := NewDatabase(memorydb.New())
tr, _ := New(common.Hash{}, triedb)
values := make(map[string]string) // tracks content of the trie
@ -540,7 +542,7 @@ func benchGet(b *testing.B, commit bool) {
b.StopTimer()
if commit {
ldb := trie.db.diskdb.(*ethdb.LDBDatabase)
ldb := trie.db.diskdb.(*leveldb.LevelDBDatabase)
ldb.Close()
os.RemoveAll(ldb.Path())
}
@ -596,7 +598,7 @@ func tempDB() (string, *Database) {
if err != nil {
panic(fmt.Sprintf("can't create temporary directory: %v", err))
}
diskdb, err := ethdb.NewLDBDatabase(dir, 256, 0)
diskdb, err := leveldb.New(dir, 256, 0, "")
if err != nil {
panic(fmt.Sprintf("can't create temporary database: %v", err))
}