gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

core, eth: implement eth/65 transaction fetcher

Browse Source
This commit is contained in:
rjl493456442
2019-10-28 19:59:07 +08:00
committed by Péter Szilágyi
parent dcffb7777f
commit 049e17116e
15 changed files with 1344 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
eth/fetcher/fetcher.go → eth/fetcher/block_fetcher.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package fetcher contains the block announcement based synchronisation.
// Package fetcher contains the announcement based blocks or transaction synchronisation.
package fetcher
import (
@ -30,13 +30,16 @@ import (
)
const (
arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block is explicitly requested
arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested
gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches
fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block
maxUncleDist = 7 // Maximum allowed backward distance from the chain head
maxQueueDist = 32 // Maximum allowed distance from the chain head to queue
hashLimit = 256 // Maximum number of unique blocks a peer may have announced
blockLimit = 64 // Maximum number of unique blocks a peer may have delivered
fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction
)
const (
maxUncleDist = 7 // Maximum allowed backward distance from the chain head
maxQueueDist = 32 // Maximum allowed distance from the chain head to queue
hashLimit = 256 // Maximum number of unique blocks a peer may have announced
blockLimit = 64 // Maximum number of unique blocks a peer may have delivered
)
var (
@ -67,9 +70,9 @@ type chainInsertFn func(types.Blocks) (int, error)
// peerDropFn is a callback type for dropping a peer detected as malicious.
type peerDropFn func(id string)
// announce is the hash notification of the availability of a new block in the
// blockAnnounce is the hash notification of the availability of a new block in the
// network.
type announce struct {
type blockAnnounce struct {
hash common.Hash // Hash of the block being announced
number uint64 // Number of the block being announced (0 = unknown | old protocol)
header *types.Header // Header of the block partially reassembled (new protocol)
@ -97,18 +100,18 @@ type bodyFilterTask struct {
time time.Time // Arrival time of the blocks' contents
}
// inject represents a schedules import operation.
type inject struct {
// blockInject represents a schedules import operation.
type blockInject struct {
origin string
block *types.Block
}
// Fetcher is responsible for accumulating block announcements from various peers
// BlockFetcher is responsible for accumulating block announcements from various peers
// and scheduling them for retrieval.
type Fetcher struct {
type BlockFetcher struct {
// Various event channels
notify chan *announce
inject chan *inject
notify chan *blockAnnounce
inject chan *blockInject
headerFilter chan chan *headerFilterTask
bodyFilter chan chan *bodyFilterTask
@ -117,16 +120,16 @@ type Fetcher struct {
quit chan struct{}
// Announce states
announces map[string]int // Per peer announce counts to prevent memory exhaustion
announced map[common.Hash][]*announce // Announced blocks, scheduled for fetching
fetching map[common.Hash]*announce // Announced blocks, currently fetching
fetched map[common.Hash][]*announce // Blocks with headers fetched, scheduled for body retrieval
completing map[common.Hash]*announce // Blocks with headers, currently body-completing
announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion
announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching
fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching
fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval
completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing
// Block cache
queue *prque.Prque // Queue containing the import operations (block number sorted)
queues map[string]int // Per peer block counts to prevent memory exhaustion
queued map[common.Hash]*inject // Set of already queued blocks (to dedupe imports)
queue *prque.Prque // Queue containing the import operations (block number sorted)
queues map[string]int // Per peer block counts to prevent memory exhaustion
queued map[common.Hash]*blockInject // Set of already queued blocks (to dedupe imports)
// Callbacks
getBlock blockRetrievalFn // Retrieves a block from the local chain
@ -137,30 +140,30 @@ type Fetcher struct {
dropPeer peerDropFn // Drops a peer for misbehaving
// Testing hooks
announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the announce list
announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list
queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue
fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch
completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62)
importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62)
}
// New creates a block fetcher to retrieve blocks based on hash announcements.
func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, dropPeer peerDropFn) *Fetcher {
return &Fetcher{
notify: make(chan *announce),
inject: make(chan *inject),
// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements.
func NewBlockFetcher(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher {
return &BlockFetcher{
notify: make(chan *blockAnnounce),
inject: make(chan *blockInject),
headerFilter: make(chan chan *headerFilterTask),
bodyFilter: make(chan chan *bodyFilterTask),
done: make(chan common.Hash),
quit: make(chan struct{}),
announces: make(map[string]int),
announced: make(map[common.Hash][]*announce),
fetching: make(map[common.Hash]*announce),
fetched: make(map[common.Hash][]*announce),
completing: make(map[common.Hash]*announce),
announced: make(map[common.Hash][]*blockAnnounce),
fetching: make(map[common.Hash]*blockAnnounce),
fetched: make(map[common.Hash][]*blockAnnounce),
completing: make(map[common.Hash]*blockAnnounce),
queue: prque.New(nil),
queues: make(map[string]int),
queued: make(map[common.Hash]*inject),
queued: make(map[common.Hash]*blockInject),
getBlock: getBlock,
verifyHeader: verifyHeader,
broadcastBlock: broadcastBlock,
@ -172,21 +175,21 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc
// Start boots up the announcement based synchroniser, accepting and processing
// hash notifications and block fetches until termination requested.
func (f *Fetcher) Start() {
func (f *BlockFetcher) Start() {
go f.loop()
}
// Stop terminates the announcement based synchroniser, canceling all pending
// operations.
func (f *Fetcher) Stop() {
func (f *BlockFetcher) Stop() {
close(f.quit)
}
// Notify announces the fetcher of the potential availability of a new block in
// the network.
func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time,
func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time,
headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error {
block := &announce{
block := &blockAnnounce{
hash: hash,
number: number,
time: time,
@ -203,8 +206,8 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time
}
// Enqueue tries to fill gaps the fetcher's future import queue.
func (f *Fetcher) Enqueue(peer string, block *types.Block) error {
op := &inject{
func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error {
op := &blockInject{
origin: peer,
block: block,
}
@ -218,7 +221,7 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error {
// FilterHeaders extracts all the headers that were explicitly requested by the fetcher,
// returning those that should be handled differently.
func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header {
func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header {
log.Trace("Filtering headers", "peer", peer, "headers", len(headers))
// Send the filter channel to the fetcher
@ -246,7 +249,7 @@ func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time.
// FilterBodies extracts all the block bodies that were explicitly requested by
// the fetcher, returning those that should be handled differently.
func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) {
func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) {
log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles))
// Send the filter channel to the fetcher
@ -274,7 +277,7 @@ func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction,
// Loop is the main fetcher loop, checking and processing various notification
// events.
func (f *Fetcher) loop() {
func (f *BlockFetcher) loop() {
// Iterate the block fetching until a quit is requested
fetchTimer := time.NewTimer(0)
completeTimer := time.NewTimer(0)
@ -289,7 +292,7 @@ func (f *Fetcher) loop() {
// Import any queued blocks that could potentially fit
height := f.chainHeight()
for !f.queue.Empty() {
op := f.queue.PopItem().(*inject)
op := f.queue.PopItem().(*blockInject)
hash := op.block.Hash()
if f.queueChangeHook != nil {
f.queueChangeHook(hash, false)
@ -313,24 +316,24 @@ func (f *Fetcher) loop() {
// Wait for an outside event to occur
select {
case <-f.quit:
// Fetcher terminating, abort all operations
// BlockFetcher terminating, abort all operations
return
case notification := <-f.notify:
// A block was announced, make sure the peer isn't DOSing us
propAnnounceInMeter.Mark(1)
blockAnnounceInMeter.Mark(1)
count := f.announces[notification.origin] + 1
if count > hashLimit {
log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit)
propAnnounceDOSMeter.Mark(1)
blockAnnounceDOSMeter.Mark(1)
break
}
// If we have a valid block number, check that it's potentially useful
if notification.number > 0 {
if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist {
log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist)
propAnnounceDropMeter.Mark(1)
blockAnnounceDropMeter.Mark(1)
break
}
}
@ -352,7 +355,7 @@ func (f *Fetcher) loop() {
case op := <-f.inject:
// A direct block insertion was requested, try and fill any pending gaps
propBroadcastInMeter.Mark(1)
blockBroadcastInMeter.Mark(1)
f.enqueue(op.origin, op.block)
case hash := <-f.done:
@ -439,7 +442,7 @@ func (f *Fetcher) loop() {
// Split the batch of headers into unknown ones (to return to the caller),
// known incomplete ones (requiring body retrievals) and completed blocks.
unknown, incomplete, complete := []*types.Header{}, []*announce{}, []*types.Block{}
unknown, incomplete, complete := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}
for _, header := range task.headers {
hash := header.Hash()
@ -475,7 +478,7 @@ func (f *Fetcher) loop() {
f.forgetHash(hash)
}
} else {
// Fetcher doesn't know about it, add to the return list
// BlockFetcher doesn't know about it, add to the return list
unknown = append(unknown, header)
}
}
@ -562,8 +565,8 @@ func (f *Fetcher) loop() {
}
}
// rescheduleFetch resets the specified fetch timer to the next announce timeout.
func (f *Fetcher) rescheduleFetch(fetch *time.Timer) {
// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout.
func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) {
// Short circuit if no blocks are announced
if len(f.announced) == 0 {
return
@ -579,7 +582,7 @@ func (f *Fetcher) rescheduleFetch(fetch *time.Timer) {
}
// rescheduleComplete resets the specified completion timer to the next fetch timeout.
func (f *Fetcher) rescheduleComplete(complete *time.Timer) {
func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) {
// Short circuit if no headers are fetched
if len(f.fetched) == 0 {
return
@ -596,27 +599,27 @@ func (f *Fetcher) rescheduleComplete(complete *time.Timer) {
// enqueue schedules a new future import operation, if the block to be imported
// has not yet been seen.
func (f *Fetcher) enqueue(peer string, block *types.Block) {
func (f *BlockFetcher) enqueue(peer string, block *types.Block) {
hash := block.Hash()
// Ensure the peer isn't DOSing us
count := f.queues[peer] + 1
if count > blockLimit {
log.Debug("Discarded propagated block, exceeded allowance", "peer", peer, "number", block.Number(), "hash", hash, "limit", blockLimit)
propBroadcastDOSMeter.Mark(1)
blockBroadcastDOSMeter.Mark(1)
f.forgetHash(hash)
return
}
// Discard any past or too distant blocks
if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist {
log.Debug("Discarded propagated block, too far away", "peer", peer, "number", block.Number(), "hash", hash, "distance", dist)
propBroadcastDropMeter.Mark(1)
blockBroadcastDropMeter.Mark(1)
f.forgetHash(hash)
return
}
// Schedule the block for future importing
if _, ok := f.queued[hash]; !ok {
op := &inject{
op := &blockInject{
origin: peer,
block: block,
}
@ -633,7 +636,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) {
// insert spawns a new goroutine to run a block insertion into the chain. If the
// block's number is at the same height as the current import phase, it updates
// the phase states accordingly.
func (f *Fetcher) insert(peer string, block *types.Block) {
func (f *BlockFetcher) insert(peer string, block *types.Block) {
hash := block.Hash()
// Run the import on a new thread
@ -651,7 +654,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
switch err := f.verifyHeader(block.Header()); err {
case nil:
// All ok, quickly propagate to our peers
propBroadcastOutTimer.UpdateSince(block.ReceivedAt)
blockBroadcastOutTimer.UpdateSince(block.ReceivedAt)
go f.broadcastBlock(block, true)
case consensus.ErrFutureBlock:
@ -669,7 +672,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
return
}
// If import succeeded, broadcast the block
propAnnounceOutTimer.UpdateSince(block.ReceivedAt)
blockAnnounceOutTimer.UpdateSince(block.ReceivedAt)
go f.broadcastBlock(block, false)
// Invoke the testing hook if needed
@ -681,7 +684,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
// forgetHash removes all traces of a block announcement from the fetcher's
// internal state.
func (f *Fetcher) forgetHash(hash common.Hash) {
func (f *BlockFetcher) forgetHash(hash common.Hash) {
// Remove all pending announces and decrement DOS counters
for _, announce := range f.announced[hash] {
f.announces[announce.origin]--
@ -723,7 +726,7 @@ func (f *Fetcher) forgetHash(hash common.Hash) {
// forgetBlock removes all traces of a queued block from the fetcher's internal
// state.
func (f *Fetcher) forgetBlock(hash common.Hash) {
func (f *BlockFetcher) forgetBlock(hash common.Hash) {
if insert := f.queued[hash]; insert != nil {
f.queues[insert.origin]--
if f.queues[insert.origin] == 0 {

4
eth/fetcher/fetcher_test.go → eth/fetcher/block_fetcher_test.go
View File

@ -76,7 +76,7 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common
// fetcherTester is a test simulator for mocking out local block chain.
type fetcherTester struct {
fetcher *Fetcher
fetcher *BlockFetcher
hashes []common.Hash // Hash chain belonging to the tester
blocks map[common.Hash]*types.Block // Blocks belonging to the tester
@ -92,7 +92,7 @@ func newTester() *fetcherTester {
blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis},
drops: make(map[string]bool),
}
tester.fetcher = New(tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer)
tester.fetcher = NewBlockFetcher(tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer)
tester.fetcher.Start()
return tester

27
eth/fetcher/metrics.go
View File

@ -23,15 +23,15 @@ import (
)
var (
propAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil)
propAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil)
propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil)
propAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil)
blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/in", nil)
blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/announces/out", nil)
blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/drop", nil)
blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/dos", nil)
propBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil)
propBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil)
propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil)
propBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil)
blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/in", nil)
blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/broadcasts/out", nil)
blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/drop", nil)
blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/dos", nil)
headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil)
bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil)
@ -40,4 +40,15 @@ var (
headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil)
bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil)
bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil)
txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/in", nil)
txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/dos", nil)
txAnnounceSkipMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/skip", nil)
txAnnounceUnderpriceMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/underprice", nil)
txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/broadcasts/in", nil)
txFetchOutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/out", nil)
txFetchSuccessMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/success", nil)
txFetchTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/timeout", nil)
txFetchInvalidMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/invalid", nil)
txFetchDurationTimer = metrics.NewRegisteredTimer("eth/fetcher/fetch/transaction/duration", nil)
)

319
eth/fetcher/tx_fetcher.go Normal file
View File

@ -0,0 +1,319 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"math/rand"
"time"
mapset "github.com/deckarep/golang-set"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
)
var (
// txAnnounceLimit is the maximum number of unique transaction a peer
// can announce in a short time.
txAnnounceLimit = 4096
// txFetchTimeout is the maximum allotted time to return an explicitly
// requested transaction.
txFetchTimeout = 5 * time.Second
// MaxTransactionFetch is the maximum transaction number can be fetched
// in one request. The rationale to pick this value is:
// In eth protocol, the softResponseLimit is 2MB. Nowdays according to
// Etherscan the average transaction size is around 200B, so in theory
// we can include lots of transaction in a single protocol packet. However
// the maximum size of a single transaction is raised to 128KB, so pick
// a middle value here to ensure we can maximize the efficiency of the
// retrieval and response size overflow won't happen in most cases.
MaxTransactionFetch = 256
// underpriceSetSize is the size of underprice set which used for maintaining
// the set of underprice transactions.
underpriceSetSize = 4096
)
// txAnnounce is the notification of the availability of a single
// new transaction in the network.
type txAnnounce struct {
origin string // Identifier of the peer originating the notification
time time.Time // Timestamp of the announcement
fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer
}
// txsAnnounce is the notification of the availability of a batch
// of new transactions in the network.
type txsAnnounce struct {
hashes []common.Hash // Batch of transaction hashes being announced
origin string // Identifier of the peer originating the notification
time time.Time // Timestamp of the announcement
fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer
}
// TxFetcher is responsible for retrieving new transaction based
// on the announcement.
type TxFetcher struct {
notify chan *txsAnnounce
cleanup chan []common.Hash
quit chan struct{}
// Announce states
announces map[string]int // Per peer transaction announce counts to prevent memory exhaustion
announced map[common.Hash][]*txAnnounce // Announced transactions, scheduled for fetching
fetching map[common.Hash]*txAnnounce // Announced transactions, currently fetching
underpriced mapset.Set // Transaction set whose price is too low for accepting
// Callbacks
hasTx func(common.Hash) bool // Retrieves a tx from the local txpool
addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool
dropPeer func(string) // Drop the specified peer
// Hooks
announceHook func([]common.Hash) // Hook which is called when a batch transactions are announced
importTxsHook func([]*types.Transaction) // Hook which is called when a batch of transactions are imported.
dropHook func(string) // Hook which is called when a peer is dropped
cleanupHook func([]common.Hash) // Hook which is called when internal status is cleaned
rejectUnderprice func(common.Hash) // Hook which is called when underprice transaction is rejected
}
// NewTxFetcher creates a transaction fetcher to retrieve transaction
// based on hash announcements.
func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, dropPeer func(string)) *TxFetcher {
return &TxFetcher{
notify: make(chan *txsAnnounce),
cleanup: make(chan []common.Hash),
quit: make(chan struct{}),
announces: make(map[string]int),
announced: make(map[common.Hash][]*txAnnounce),
fetching: make(map[common.Hash]*txAnnounce),
underpriced: mapset.NewSet(),
hasTx: hasTx,
addTxs: addTxs,
dropPeer: dropPeer,
}
}
// Notify announces the fetcher of the potential availability of a
// new transaction in the network.
func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fetchTxs func([]common.Hash)) error {
announce := &txsAnnounce{
hashes: hashes,
time: time,
origin: peer,
fetchTxs: fetchTxs,
}
select {
case f.notify <- announce:
return nil
case <-f.quit:
return errTerminated
}
}
// EnqueueTxs imports a batch of received transaction into fetcher.
func (f *TxFetcher) EnqueueTxs(peer string, txs []*types.Transaction) error {
var (
drop bool
hashes []common.Hash
)
errs := f.addTxs(txs)
for i, err := range errs {
if err != nil {
// Drop peer if the received transaction isn't signed properly.
drop = (drop || err == core.ErrInvalidSender)
txFetchInvalidMeter.Mark(1)
// Track the transaction hash if the price is too low for us.
// Avoid re-request this transaction when we receive another
// announcement.
if err == core.ErrUnderpriced {
for f.underpriced.Cardinality() >= underpriceSetSize {
f.underpriced.Pop()
}
f.underpriced.Add(txs[i].Hash())
}
}
hashes = append(hashes, txs[i].Hash())
}
if f.importTxsHook != nil {
f.importTxsHook(txs)
}
// Drop the peer if some transaction failed signature verification.
// We can regard this peer is trying to DOS us by feeding lots of
// random hashes.
if drop {
f.dropPeer(peer)
if f.dropHook != nil {
f.dropHook(peer)
}
}
select {
case f.cleanup <- hashes:
return nil
case <-f.quit:
return errTerminated
}
}
// Start boots up the announcement based synchroniser, accepting and processing
// hash notifications and block fetches until termination requested.
func (f *TxFetcher) Start() {
go f.loop()
}
// Stop terminates the announcement based synchroniser, canceling all pending
// operations.
func (f *TxFetcher) Stop() {
close(f.quit)
}
func (f *TxFetcher) loop() {
fetchTimer := time.NewTimer(0)
for {
// Clean up any expired transaction fetches.
// There are many cases can lead to it:
// * We send the request to busy peer which can reply immediately
// * We send the request to malicious peer which doesn't reply deliberately
// * We send the request to normal peer for a batch of transaction, but some
// transactions have been included into blocks. According to EIP these txs
// won't be included.
// But it's fine to delete the fetching record and reschedule fetching iff we
// receive the annoucement again.
for hash, announce := range f.fetching {
if time.Since(announce.time) > txFetchTimeout {
delete(f.fetching, hash)
txFetchTimeoutMeter.Mark(1)
}
}
select {
case anno := <-f.notify:
txAnnounceInMeter.Mark(int64(len(anno.hashes)))
// Drop the new announce if there are too many accumulated.
count := f.announces[anno.origin] + len(anno.hashes)
if count > txAnnounceLimit {
txAnnounceDOSMeter.Mark(int64(count - txAnnounceLimit))
break
}
f.announces[anno.origin] = count
// All is well, schedule the announce if transaction is not yet downloading
empty := len(f.announced) == 0
for _, hash := range anno.hashes {
if _, ok := f.fetching[hash]; ok {
continue
}
if f.underpriced.Contains(hash) {
txAnnounceUnderpriceMeter.Mark(1)
if f.rejectUnderprice != nil {
f.rejectUnderprice(hash)
}
continue
}
f.announced[hash] = append(f.announced[hash], &txAnnounce{
origin: anno.origin,
time: anno.time,
fetchTxs: anno.fetchTxs,
})
}
if empty && len(f.announced) > 0 {
f.reschedule(fetchTimer)
}
if f.announceHook != nil {
f.announceHook(anno.hashes)
}
case <-fetchTimer.C:
// At least one tx's timer ran out, check for needing retrieval
request := make(map[string][]common.Hash)
for hash, announces := range f.announced {
if time.Since(announces[0].time) > arriveTimeout-gatherSlack {
// Pick a random peer to retrieve from, reset all others
announce := announces[rand.Intn(len(announces))]
f.forgetHash(hash)
// Skip fetching if we already receive the transaction.
if f.hasTx(hash) {
txAnnounceSkipMeter.Mark(1)
continue
}
// If the transaction still didn't arrive, queue for fetching
request[announce.origin] = append(request[announce.origin], hash)
f.fetching[hash] = announce
}
}
// Send out all block header requests
for peer, hashes := range request {
log.Trace("Fetching scheduled transactions", "peer", peer, "txs", hashes)
fetchTxs := f.fetching[hashes[0]].fetchTxs
fetchTxs(hashes)
txFetchOutMeter.Mark(int64(len(hashes)))
}
// Schedule the next fetch if blocks are still pending
f.reschedule(fetchTimer)
case hashes := <-f.cleanup:
for _, hash := range hashes {
f.forgetHash(hash)
anno, exist := f.fetching[hash]
if !exist {
txBroadcastInMeter.Mark(1) // Directly transaction propagation
continue
}
txFetchDurationTimer.UpdateSince(anno.time)
txFetchSuccessMeter.Mark(1)
delete(f.fetching, hash)
}
if f.cleanupHook != nil {
f.cleanupHook(hashes)
}
case <-f.quit:
return
}
}
}
// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout.
func (f *TxFetcher) reschedule(fetch *time.Timer) {
// Short circuit if no transactions are announced
if len(f.announced) == 0 {
return
}
// Otherwise find the earliest expiring announcement
earliest := time.Now()
for _, announces := range f.announced {
if earliest.After(announces[0].time) {
earliest = announces[0].time
}
}
fetch.Reset(arriveTimeout - time.Since(earliest))
}
func (f *TxFetcher) forgetHash(hash common.Hash) {
// Remove all pending announces and decrement DOS counters
for _, announce := range f.announced[hash] {
f.announces[announce.origin]--
if f.announces[announce.origin] <= 0 {
delete(f.announces, announce.origin)
}
}
delete(f.announced, hash)
}

318
eth/fetcher/tx_fetcher_test.go Normal file
View File

@ -0,0 +1,318 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"crypto/ecdsa"
"math/big"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
)
func init() {
rand.Seed(int64(time.Now().Nanosecond()))
txAnnounceLimit = 64
MaxTransactionFetch = 16
}
func makeTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction {
var txs []*types.Transaction
for i := 0; i < target; i++ {
random := rand.Uint32()
tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil)
tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), key)
txs = append(txs, tx)
}
return txs
}
func makeUnsignedTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction {
var txs []*types.Transaction
for i := 0; i < target; i++ {
random := rand.Uint32()
tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil)
txs = append(txs, tx)
}
return txs
}
type txfetcherTester struct {
fetcher *TxFetcher
priceLimit *big.Int
sender *ecdsa.PrivateKey
senderAddr common.Address
signer types.Signer
txs map[common.Hash]*types.Transaction
dropped map[string]struct{}
lock sync.RWMutex
}
func newTxFetcherTester() *txfetcherTester {
key, _ := crypto.GenerateKey()
addr := crypto.PubkeyToAddress(key.PublicKey)
t := &txfetcherTester{
sender: key,
senderAddr: addr,
signer: types.NewEIP155Signer(big.NewInt(1)),
txs: make(map[common.Hash]*types.Transaction),
dropped: make(map[string]struct{}),
}
t.fetcher = NewTxFetcher(t.hasTx, t.addTxs, t.dropPeer)
t.fetcher.Start()
return t
}
func (t *txfetcherTester) hasTx(hash common.Hash) bool {
t.lock.RLock()
defer t.lock.RUnlock()
return t.txs[hash] != nil
}
func (t *txfetcherTester) addTxs(txs []*types.Transaction) []error {
t.lock.Lock()
defer t.lock.Unlock()
var errors []error
for _, tx := range txs {
// Make sure the transaction is signed properly
_, err := types.Sender(t.signer, tx)
if err != nil {
errors = append(errors, core.ErrInvalidSender)
continue
}
// Make sure the price is high enough to accpet
if t.priceLimit != nil && tx.GasPrice().Cmp(t.priceLimit) < 0 {
errors = append(errors, core.ErrUnderpriced)
continue
}
t.txs[tx.Hash()] = tx
errors = append(errors, nil)
}
return errors
}
func (t *txfetcherTester) dropPeer(id string) {
t.lock.Lock()
defer t.lock.Unlock()
t.dropped[id] = struct{}{}
}
// makeTxFetcher retrieves a batch of transaction associated with a simulated peer.
func (t *txfetcherTester) makeTxFetcher(peer string, txs []*types.Transaction) func(hashes []common.Hash) {
closure := make(map[common.Hash]*types.Transaction)
for _, tx := range txs {
closure[tx.Hash()] = tx
}
return func(hashes []common.Hash) {
var txs []*types.Transaction
for _, hash := range hashes {
tx := closure[hash]
if tx == nil {
continue
}
txs = append(txs, tx)
}
// Return on a new thread
go t.fetcher.EnqueueTxs(peer, txs)
}
}
func TestSequentialTxAnnouncements(t *testing.T) {
tester := newTxFetcherTester()
txs := makeTransactions(tester.sender, txAnnounceLimit)
retrieveTxs := tester.makeTxFetcher("peer", txs)
newTxsCh := make(chan struct{})
tester.fetcher.importTxsHook = func(transactions []*types.Transaction) {
newTxsCh <- struct{}{}
}
for _, tx := range txs {
tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), retrieveTxs)
select {
case <-newTxsCh:
case <-time.NewTimer(time.Second).C:
t.Fatalf("timeout")
}
}
if len(tester.txs) != len(txs) {
t.Fatalf("Imported transaction number mismatch, want %d, got %d", len(txs), len(tester.txs))
}
}
func TestConcurrentAnnouncements(t *testing.T) {
tester := newTxFetcherTester()
txs := makeTransactions(tester.sender, txAnnounceLimit)
txFetcherFn1 := tester.makeTxFetcher("peer1", txs)
txFetcherFn2 := tester.makeTxFetcher("peer2", txs)
var (
count uint32
done = make(chan struct{})
)
tester.fetcher.importTxsHook = func(transactions []*types.Transaction) {
atomic.AddUint32(&count, uint32(len(transactions)))
if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) {
done <- struct{}{}
}
}
for _, tx := range txs {
tester.fetcher.Notify("peer1", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn1)
tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout+time.Millisecond), txFetcherFn2)
tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout-time.Millisecond), txFetcherFn2)
}
select {
case <-done:
case <-time.NewTimer(time.Second).C:
t.Fatalf("timeout")
}
}
func TestBatchAnnouncements(t *testing.T) {
tester := newTxFetcherTester()
txs := makeTransactions(tester.sender, txAnnounceLimit)
retrieveTxs := tester.makeTxFetcher("peer", txs)
var count uint32
var done = make(chan struct{})
tester.fetcher.importTxsHook = func(txs []*types.Transaction) {
atomic.AddUint32(&count, uint32(len(txs)))
if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) {
done <- struct{}{}
}
}
// Send all announces which exceeds the limit.
var hashes []common.Hash
for _, tx := range txs {
hashes = append(hashes, tx.Hash())
}
tester.fetcher.Notify("peer", hashes, time.Now(), retrieveTxs)
select {
case <-done:
case <-time.NewTimer(time.Second).C:
t.Fatalf("timeout")
}
}
func TestPropagationAfterAnnounce(t *testing.T) {
tester := newTxFetcherTester()
txs := makeTransactions(tester.sender, txAnnounceLimit)
var cleaned = make(chan struct{})
tester.fetcher.cleanupHook = func(hashes []common.Hash) {
cleaned <- struct{}{}
}
retrieveTxs := tester.makeTxFetcher("peer", txs)
for _, tx := range txs {
tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), retrieveTxs)
tester.fetcher.EnqueueTxs("peer", []*types.Transaction{tx})
// It's ok to read the map directly since no write
// will happen in the same time.
<-cleaned
if len(tester.fetcher.announced) != 0 {
t.Fatalf("Announcement should be cleaned, got %d", len(tester.fetcher.announced))
}
}
}
func TestEnqueueTransactions(t *testing.T) {
tester := newTxFetcherTester()
txs := makeTransactions(tester.sender, txAnnounceLimit)
done := make(chan struct{})
tester.fetcher.importTxsHook = func(transactions []*types.Transaction) {
if len(transactions) == txAnnounceLimit {
done <- struct{}{}
}
}
go tester.fetcher.EnqueueTxs("peer", txs)
select {
case <-done:
case <-time.NewTimer(time.Second).C:
t.Fatalf("timeout")
}
}
func TestInvalidTxAnnounces(t *testing.T) {
tester := newTxFetcherTester()
var txs []*types.Transaction
txs = append(txs, makeUnsignedTransactions(tester.sender, 1)...)
txs = append(txs, makeTransactions(tester.sender, 1)...)
txFetcherFn := tester.makeTxFetcher("peer", txs)
dropped := make(chan string, 1)
tester.fetcher.dropHook = func(s string) { dropped <- s }
for _, tx := range txs {
tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), txFetcherFn)
}
select {
case s := <-dropped:
if s != "peer" {
t.Fatalf("invalid dropped peer")
}
case <-time.NewTimer(time.Second).C:
t.Fatalf("timeout")
}
}
func TestRejectUnderpriced(t *testing.T) {
tester := newTxFetcherTester()
tester.priceLimit = big.NewInt(10000)
done := make(chan struct{})
tester.fetcher.importTxsHook = func([]*types.Transaction) { done <- struct{}{} }
reject := make(chan struct{})
tester.fetcher.rejectUnderprice = func(common.Hash) { reject <- struct{}{} }
tx := types.NewTransaction(0, common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(100)), 100, big.NewInt(int64(100)), nil)
tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), tester.sender)
txFetcherFn := tester.makeTxFetcher("peer", []*types.Transaction{tx})
// Send the announcement first time
tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn)
<-done
// Resend the announcement, shouldn't schedule fetching this time
tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn)
select {
case <-reject:
case <-time.NewTimer(time.Second).C:
t.Fatalf("timeout")
}
}