gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

simplification of Fetchers (#1344)

Browse Source
This commit is contained in:
Anton Evangelatov
2019-06-17 10:01:12 +02:00
committed by GitHub
parent 21adbe2fac
commit 0b724bd4d5
23 changed files with 746 additions and 2246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
storage/feed/handler.go
View File

@ -192,12 +192,13 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error)
ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout)
defer cancel()
ch, err := h.chunkStore.Get(ctx, chunk.ModeGetLookup, id.Addr())
r := storage.NewRequest(id.Addr())
ch, err := h.chunkStore.Get(ctx, chunk.ModeGetLookup, r)
if err != nil {
if err == context.DeadlineExceeded { // chunk not found
if err == context.DeadlineExceeded || err == storage.ErrNoSuitablePeer { // chunk not found
return nil, nil
}
return nil, err //something else happened or context was cancelled.
return nil, err
}
var request Request

2
storage/feed/lookup/lookup.go
View File

@ -58,6 +58,8 @@ var TimeAfter = time.After
// It should return <nil> if a value is found, but its timestamp is higher than "now"
// It should only return an error in case the handler wants to stop the
// lookup process entirely.
// If the context is canceled, it must return context.Canceled
type ReadFunc func(ctx context.Context, epoch Epoch, now uint64) (interface{}, error)
// NoClue is a hint that can be provided when the Lookup caller does not have

20
storage/feed/testutil.go
View File

@ -18,8 +18,8 @@ package feed
import (
"context"
"errors"
"path/filepath"
"sync"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethersphere/swarm/chunk"
@ -39,17 +39,6 @@ func (t *TestHandler) Close() {
t.chunkStore.Close()
}
type mockNetFetcher struct{}
func (m *mockNetFetcher) Request(hopCount uint8) {
}
func (m *mockNetFetcher) Offer(source *enode.ID) {
}
func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetFetcher {
return &mockNetFetcher{}
}
// NewTestHandler creates Handler object to be used for testing purposes.
func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) {
path := filepath.Join(datadir, testDbDirName)
@ -62,11 +51,10 @@ func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error)
localStore := chunk.NewValidatorStore(db, storage.NewContentAddressValidator(storage.MakeHashFunc(feedsHashAlgorithm)), fh)
netStore, err := storage.NewNetStore(localStore, nil)
if err != nil {
return nil, err
netStore := storage.NewNetStore(localStore, enode.ID{})
netStore.RemoteGet = func(ctx context.Context, req *storage.Request, localID enode.ID) (*enode.ID, error) {
return nil, errors.New("not found")
}
netStore.NewNetFetcherFunc = newFakeNetFetcher
fh.SetStore(netStore)
return &TestHandler{fh}, nil
}

5
storage/filestore.go
View File

@ -39,9 +39,8 @@ implementation for storage or retrieval.
*/
const (
defaultLDBCapacity = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB
defaultCacheCapacity = 10000 // capacity for in-memory chunks' cache
defaultChunkRequestsCacheCapacity = 5000000 // capacity for container holding outgoing requests for chunks. should be set to LevelDB capacity
defaultLDBCapacity = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB
defaultCacheCapacity = 10000 // capacity for in-memory chunks' cache
)
type FileStore struct {

46
storage/lnetstore.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"context"
"github.com/ethersphere/swarm/chunk"
"github.com/ethersphere/swarm/network/timeouts"
)
// LNetStore is a wrapper of NetStore, which implements the chunk.Store interface. It is used only by the FileStore,
// the component used by the Swarm API to store and retrieve content and to split and join chunks.
type LNetStore struct {
*NetStore
}
// NewLNetStore is a constructor for LNetStore
func NewLNetStore(store *NetStore) *LNetStore {
return &LNetStore{
NetStore: store,
}
}
// Get converts a chunk reference to a chunk Request (with empty Origin), handled by the NetStore, and
// returns the requested chunk, or error.
func (n *LNetStore) Get(ctx context.Context, mode chunk.ModeGet, ref Address) (ch Chunk, err error) {
ctx, cancel := context.WithTimeout(ctx, timeouts.FetcherGlobalTimeout)
defer cancel()
return n.NetStore.Get(ctx, mode, NewRequest(ref))
}

485
storage/netstore.go
View File

@ -18,318 +18,291 @@ package storage
import (
"context"
"encoding/hex"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethersphere/swarm/chunk"
"github.com/ethersphere/swarm/log"
"github.com/ethersphere/swarm/network/timeouts"
"github.com/ethersphere/swarm/spancontext"
"github.com/opentracing/opentracing-go"
lru "github.com/hashicorp/golang-lru"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p/enode"
olog "github.com/opentracing/opentracing-go/log"
"github.com/syndtr/goleveldb/leveldb"
lru "github.com/hashicorp/golang-lru"
"golang.org/x/sync/singleflight"
)
type (
NewNetFetcherFunc func(ctx context.Context, addr Address, peers *sync.Map) NetFetcher
const (
// capacity for the fetchers LRU cache
fetchersCapacity = 500000
)
type NetFetcher interface {
Request(hopCount uint8)
Offer(source *enode.ID)
var (
ErrNoSuitablePeer = errors.New("no suitable peer")
)
// Fetcher is a struct which maintains state of remote requests.
// Fetchers are stored in fetchers map and signal to all interested parties if a given chunk is delivered
// the mutex controls who closes the channel, and make sure we close the channel only once
type Fetcher struct {
Delivered chan struct{} // when closed, it means that the chunk this Fetcher refers to is delivered
// it is possible for multiple actors to be delivering the same chunk,
// for example through syncing and through retrieve request. however we want the `Delivered` channel to be closed only
// once, even if we put the same chunk multiple times in the NetStore.
once sync.Once
CreatedAt time.Time // timestamp when the fetcher was created, used for metrics measuring lifetime of fetchers
CreatedBy string // who created the fetcher - "request" or "syncing", used for metrics measuring lifecycle of fetchers
RequestedBySyncer bool // whether we have issued at least once a request through Offered/Wanted hashes flow
}
// NetStore is an extension of local storage
// NewFetcher is a constructor for a Fetcher
func NewFetcher() *Fetcher {
return &Fetcher{make(chan struct{}), sync.Once{}, time.Now(), "", false}
}
// SafeClose signals to interested parties (those waiting for a signal on fi.Delivered) that a chunk is delivered.
// It closes the fi.Delivered channel through the sync.Once object, because it is possible for a chunk to be
// delivered multiple times concurrently.
func (fi *Fetcher) SafeClose() {
fi.once.Do(func() {
close(fi.Delivered)
})
}
type RemoteGetFunc func(ctx context.Context, req *Request, localID enode.ID) (*enode.ID, error)
// NetStore is an extension of LocalStore
// it implements the ChunkStore interface
// on request it initiates remote cloud retrieval using a fetcher
// fetchers are unique to a chunk and are stored in fetchers LRU memory cache
// fetchFuncFactory is a factory object to create a fetch function for a specific chunk address
// on request it initiates remote cloud retrieval
type NetStore struct {
chunk.Store
mu sync.Mutex
fetchers *lru.Cache
NewNetFetcherFunc NewNetFetcherFunc
closeC chan struct{}
localID enode.ID // our local enode - used when issuing RetrieveRequests
fetchers *lru.Cache
putMu sync.Mutex
requestGroup singleflight.Group
RemoteGet RemoteGetFunc
}
var fetcherTimeout = 2 * time.Minute // timeout to cancel the fetcher even if requests are coming in
// NewNetStore creates a new NetStore using the provided chunk.Store and localID of the node.
func NewNetStore(store chunk.Store, localID enode.ID) *NetStore {
fetchers, _ := lru.New(fetchersCapacity)
// NewNetStore creates a new NetStore object using the given local store. newFetchFunc is a
// constructor function that can create a fetch function for a specific chunk address.
func NewNetStore(store chunk.Store, nnf NewNetFetcherFunc) (*NetStore, error) {
fetchers, err := lru.New(defaultChunkRequestsCacheCapacity)
if err != nil {
return nil, err
}
return &NetStore{
Store: store,
fetchers: fetchers,
NewNetFetcherFunc: nnf,
closeC: make(chan struct{}),
}, nil
fetchers: fetchers,
Store: store,
localID: localID,
}
}
// Put stores a chunk in localstore, and delivers to all requestor peers using the fetcher stored in
// the fetchers cache
func (n *NetStore) Put(ctx context.Context, mode chunk.ModePut, ch Chunk) (bool, error) {
n.mu.Lock()
defer n.mu.Unlock()
n.putMu.Lock()
defer n.putMu.Unlock()
// put to the chunk to the store, there should be no error
log.Trace("netstore.put", "ref", ch.Address().String(), "mode", mode)
// put the chunk to the localstore, there should be no error
exists, err := n.Store.Put(ctx, mode, ch)
if err != nil {
return exists, err
}
// if chunk is now put in the store, check if there was an active fetcher and call deliver on it
// (this delivers the chunk to requestors via the fetcher)
log.Trace("n.getFetcher", "ref", ch.Address())
if f := n.getFetcher(ch.Address()); f != nil {
log.Trace("n.getFetcher deliver", "ref", ch.Address())
f.deliver(ctx, ch)
// notify RemoteGet (or SwarmSyncerClient) about a chunk delivery and it being stored
fi, ok := n.fetchers.Get(ch.Address().String())
if ok {
// we need SafeClose, because it is possible for a chunk to both be
// delivered through syncing and through a retrieve request
fii := fi.(*Fetcher)
fii.SafeClose()
log.Trace("netstore.put chunk delivered and stored", "ref", ch.Address().String())
metrics.GetOrRegisterResettingTimer(fmt.Sprintf("netstore.fetcher.lifetime.%s", fii.CreatedBy), nil).UpdateSince(fii.CreatedAt)
// helper snippet to log if a chunk took way to long to be delivered
slowChunkDeliveryThreshold := 5 * time.Second
if time.Since(fii.CreatedAt) > slowChunkDeliveryThreshold {
log.Trace("netstore.put slow chunk delivery", "ref", ch.Address().String())
}
n.fetchers.Remove(ch.Address().String())
}
return exists, nil
}
// Get retrieves the chunk from the NetStore DPA synchronously.
// It calls NetStore.get, and if the chunk is not in local Storage
// it calls fetch with the request, which blocks until the chunk
// arrived or context is done
func (n *NetStore) Get(rctx context.Context, mode chunk.ModeGet, ref Address) (Chunk, error) {
chunk, fetch, err := n.get(rctx, mode, ref)
if err != nil {
return nil, err
}
if chunk != nil {
// this is not measuring how long it takes to get the chunk for the localstore, but
// rather just adding a span for clarity when inspecting traces in Jaeger, in order
// to make it easier to reason which is the node that actually delivered a chunk.
_, sp := spancontext.StartSpan(
rctx,
"localstore.get")
defer sp.Finish()
return chunk, nil
}
return fetch(rctx)
}
// FetchFunc returns nil if the store contains the given address. Otherwise it returns a wait function,
// which returns after the chunk is available or the context is done
func (n *NetStore) FetchFunc(ctx context.Context, ref Address) func(context.Context) error {
chunk, fetch, _ := n.get(ctx, chunk.ModeGetRequest, ref)
if chunk != nil {
return nil
}
return func(ctx context.Context) error {
_, err := fetch(ctx)
return err
}
}
// Close chunk store
func (n *NetStore) Close() (err error) {
close(n.closeC)
wg := sync.WaitGroup{}
for _, key := range n.fetchers.Keys() {
if f, ok := n.fetchers.Get(key); ok {
if fetch, ok := f.(*fetcher); ok {
wg.Add(1)
go func(fetch *fetcher) {
defer wg.Done()
fetch.cancel()
select {
case <-fetch.deliveredC:
case <-fetch.cancelledC:
}
}(fetch)
}
}
}
wg.Wait()
func (n *NetStore) Close() error {
return n.Store.Close()
}
// get attempts at retrieving the chunk from LocalStore
// If it is not found then using getOrCreateFetcher:
// 1. Either there is already a fetcher to retrieve it
// 2. A new fetcher is created and saved in the fetchers cache
// From here on, all Get will hit on this fetcher until the chunk is delivered
// or all fetcher contexts are done.
// It returns a chunk, a fetcher function and an error
// If chunk is nil, the returned fetch function needs to be called with a context to return the chunk.
func (n *NetStore) get(ctx context.Context, mode chunk.ModeGet, ref Address) (Chunk, func(context.Context) (Chunk, error), error) {
n.mu.Lock()
defer n.mu.Unlock()
// Get retrieves a chunk
// If it is not found in the LocalStore then it uses RemoteGet to fetch from the network.
func (n *NetStore) Get(ctx context.Context, mode chunk.ModeGet, req *Request) (Chunk, error) {
metrics.GetOrRegisterCounter("netstore.get", nil).Inc(1)
start := time.Now()
chunk, err := n.Store.Get(ctx, mode, ref)
ref := req.Addr
log.Trace("netstore.get", "ref", ref.String())
ch, err := n.Store.Get(ctx, mode, ref)
if err != nil {
// TODO: Fix comparison - we should be comparing against leveldb.ErrNotFound, this error should be wrapped.
// TODO: fix comparison - we should be comparing against leveldb.ErrNotFound, this error should be wrapped.
if err != ErrChunkNotFound && err != leveldb.ErrNotFound {
log.Debug("Received error from LocalStore other than ErrNotFound", "err", err)
log.Error("localstore get error", "err", err)
}
// The chunk is not available in the LocalStore, let's get the fetcher for it, or create a new one
// if it doesn't exist yet
f := n.getOrCreateFetcher(ctx, ref)
// If the caller needs the chunk, it has to use the returned fetch function to get it
return nil, f.Fetch, nil
}
return chunk, nil, nil
}
log.Trace("netstore.chunk-not-in-localstore", "ref", ref.String())
// getOrCreateFetcher attempts at retrieving an existing fetchers
// if none exists, creates one and saves it in the fetchers cache
// caller must hold the lock
func (n *NetStore) getOrCreateFetcher(ctx context.Context, ref Address) *fetcher {
if f := n.getFetcher(ref); f != nil {
return f
}
v, err, _ := n.requestGroup.Do(ref.String(), func() (interface{}, error) {
// currently we issue a retrieve request if a fetcher
// has already been created by a syncer for that particular chunk.
// so it is possible to
// have 2 in-flight requests for the same chunk - one by a
// syncer (offered/wanted/deliver flow) and one from
// here - retrieve request
fi, _, ok := n.GetOrCreateFetcher(ctx, ref, "request")
if ok {
err := n.RemoteFetch(ctx, req, fi)
if err != nil {
return nil, err
}
}
// no fetcher for the given address, we have to create a new one
key := hex.EncodeToString(ref)
// create the context during which fetching is kept alive
cctx, cancel := context.WithTimeout(ctx, fetcherTimeout)
// destroy is called when all requests finish
destroy := func() {
// remove fetcher from fetchers
n.fetchers.Remove(key)
// stop fetcher by cancelling context called when
// all requests cancelled/timedout or chunk is delivered
cancel()
}
// peers always stores all the peers which have an active request for the chunk. It is shared
// between fetcher and the NewFetchFunc function. It is needed by the NewFetchFunc because
// the peers which requested the chunk should not be requested to deliver it.
peers := &sync.Map{}
ch, err := n.Store.Get(ctx, mode, ref)
if err != nil {
log.Error(err.Error(), "ref", ref)
return nil, errors.New("item should have been in localstore, but it is not")
}
cctx, sp := spancontext.StartSpan(
cctx,
"netstore.fetcher",
)
// fi could be nil (when ok == false) if the chunk was added to the NetStore between n.store.Get and the call to n.GetOrCreateFetcher
if fi != nil {
metrics.GetOrRegisterResettingTimer(fmt.Sprintf("fetcher.%s.request", fi.CreatedBy), nil).UpdateSince(start)
}
sp.LogFields(olog.String("ref", ref.String()))
fetcher := newFetcher(sp, ref, n.NewNetFetcherFunc(cctx, ref, peers), destroy, peers, n.closeC)
n.fetchers.Add(key, fetcher)
return ch, nil
})
return fetcher
}
// getFetcher retrieves the fetcher for the given address from the fetchers cache if it exists,
// otherwise it returns nil
func (n *NetStore) getFetcher(ref Address) *fetcher {
key := hex.EncodeToString(ref)
f, ok := n.fetchers.Get(key)
if ok {
return f.(*fetcher)
}
return nil
}
// RequestsCacheLen returns the current number of outgoing requests stored in the cache
func (n *NetStore) RequestsCacheLen() int {
return n.fetchers.Len()
}
// One fetcher object is responsible to fetch one chunk for one address, and keep track of all the
// peers who have requested it and did not receive it yet.
type fetcher struct {
addr Address // address of chunk
chunk Chunk // fetcher can set the chunk on the fetcher
deliveredC chan struct{} // chan signalling chunk delivery to requests
cancelledC chan struct{} // chan signalling the fetcher has been cancelled (removed from fetchers in NetStore)
netFetcher NetFetcher // remote fetch function to be called with a request source taken from the context
cancel func() // cleanup function for the remote fetcher to call when all upstream contexts are called
peers *sync.Map // the peers which asked for the chunk
requestCnt int32 // number of requests on this chunk. If all the requests are done (delivered or context is done) the cancel function is called
deliverOnce *sync.Once // guarantees that we only close deliveredC once
span opentracing.Span // measure retrieve time per chunk
}
// newFetcher creates a new fetcher object for the fiven addr. fetch is the function which actually
// does the retrieval (in non-test cases this is coming from the network package). cancel function is
// called either
// 1. when the chunk has been fetched all peers have been either notified or their context has been done
// 2. the chunk has not been fetched but all context from all the requests has been done
// The peers map stores all the peers which have requested chunk.
func newFetcher(span opentracing.Span, addr Address, nf NetFetcher, cancel func(), peers *sync.Map, closeC chan struct{}) *fetcher {
cancelOnce := &sync.Once{} // cancel should only be called once
return &fetcher{
addr: addr,
deliveredC: make(chan struct{}),
deliverOnce: &sync.Once{},
cancelledC: closeC,
netFetcher: nf,
cancel: func() {
cancelOnce.Do(func() {
cancel()
})
},
peers: peers,
span: span,
}
}
// Fetch fetches the chunk synchronously, it is called by NetStore.Get is the chunk is not available
// locally.
func (f *fetcher) Fetch(rctx context.Context) (Chunk, error) {
atomic.AddInt32(&f.requestCnt, 1)
defer func() {
// if all the requests are done the fetcher can be cancelled
if atomic.AddInt32(&f.requestCnt, -1) == 0 {
f.cancel()
}
f.span.Finish()
}()
// The peer asking for the chunk. Store in the shared peers map, but delete after the request
// has been delivered
peer := rctx.Value("peer")
if peer != nil {
f.peers.Store(peer, time.Now())
defer f.peers.Delete(peer)
}
// If there is a source in the context then it is an offer, otherwise a request
sourceIF := rctx.Value("source")
hopCount, _ := rctx.Value("hopcount").(uint8)
if sourceIF != nil {
var source enode.ID
if err := source.UnmarshalText([]byte(sourceIF.(string))); err != nil {
if err != nil {
log.Trace(err.Error(), "ref", ref)
return nil, err
}
f.netFetcher.Offer(&source)
c := v.(Chunk)
log.Trace("netstore.singleflight returned", "ref", ref.String(), "err", err)
return c, nil
}
ctx, ssp := spancontext.StartSpan(
ctx,
"localstore.get")
defer ssp.Finish()
return ch, nil
}
// RemoteFetch is handling the retry mechanism when making a chunk request to our peers.
// For a given chunk Request, we call RemoteGet, which selects the next eligible peer and
// issues a RetrieveRequest and we wait for a delivery. If a delivery doesn't arrive within the SearchTimeout
// we retry.
func (n *NetStore) RemoteFetch(ctx context.Context, req *Request, fi *Fetcher) error {
// while we haven't timed-out, and while we don't have a chunk,
// iterate over peers and try to find a chunk
metrics.GetOrRegisterCounter("remote.fetch", nil).Inc(1)
ref := req.Addr
for {
metrics.GetOrRegisterCounter("remote.fetch.inner", nil).Inc(1)
ctx, osp := spancontext.StartSpan(
ctx,
"remote.fetch")
osp.LogFields(olog.String("ref", ref.String()))
log.Trace("remote.fetch", "ref", ref)
currentPeer, err := n.RemoteGet(ctx, req, n.localID)
if err != nil {
log.Trace(err.Error(), "ref", ref)
osp.LogFields(olog.String("err", err.Error()))
osp.Finish()
return ErrNoSuitablePeer
}
// add peer to the set of peers to skip from now
log.Trace("remote.fetch, adding peer to skip", "ref", ref, "peer", currentPeer.String())
req.PeersToSkip.Store(currentPeer.String(), time.Now())
select {
case <-fi.Delivered:
log.Trace("remote.fetch, chunk delivered", "ref", ref)
osp.LogFields(olog.Bool("delivered", true))
osp.Finish()
return nil
case <-time.After(timeouts.SearchTimeout):
metrics.GetOrRegisterCounter("remote.fetch.timeout.search", nil).Inc(1)
osp.LogFields(olog.Bool("timeout", true))
osp.Finish()
break
case <-ctx.Done(): // global fetcher timeout
log.Trace("remote.fetch, fail", "ref", ref)
metrics.GetOrRegisterCounter("remote.fetch.timeout.global", nil).Inc(1)
osp.LogFields(olog.Bool("fail", true))
osp.Finish()
return ctx.Err()
}
}
}
// Has is the storage layer entry point to query the underlying
// database to return if it has a chunk or not.
func (n *NetStore) Has(ctx context.Context, ref Address) (bool, error) {
return n.Store.Has(ctx, ref)
}
// GetOrCreateFetcher returns the Fetcher for a given chunk, if this chunk is not in the LocalStore.
// If the chunk is in the LocalStore, it returns nil for the Fetcher and ok == false
func (n *NetStore) GetOrCreateFetcher(ctx context.Context, ref Address, interestedParty string) (f *Fetcher, loaded bool, ok bool) {
n.putMu.Lock()
defer n.putMu.Unlock()
has, err := n.Store.Has(ctx, ref)
if err != nil {
log.Error(err.Error())
}
if has {
return nil, false, false
}
f = NewFetcher()
v, loaded := n.fetchers.Get(ref.String())
log.Trace("netstore.has-with-callback.loadorstore", "ref", ref.String(), "loaded", loaded)
if loaded {
f = v.(*Fetcher)
} else {
f.netFetcher.Request(hopCount)
f.CreatedBy = interestedParty
n.fetchers.Add(ref.String(), f)
}
// wait until either the chunk is delivered or the context is done
select {
case <-rctx.Done():
return nil, rctx.Err()
case <-f.deliveredC:
return f.chunk, nil
case <-f.cancelledC:
return nil, fmt.Errorf("fetcher cancelled")
// if fetcher created by request, but we get a call from syncer, make sure we issue a second request
if f.CreatedBy != interestedParty && !f.RequestedBySyncer {
f.RequestedBySyncer = true
return f, false, true
}
}
// deliver is called by NetStore.Put to notify all pending requests
func (f *fetcher) deliver(ctx context.Context, ch Chunk) {
f.deliverOnce.Do(func() {
f.chunk = ch
// closing the deliveredC channel will terminate ongoing requests
close(f.deliveredC)
log.Trace("n.getFetcher close deliveredC", "ref", ch.Address())
})
return f, loaded, true
}

702
storage/netstore_test.go
View File

@ -1,702 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"io/ioutil"
"os"
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethersphere/swarm/chunk"
"github.com/ethersphere/swarm/storage/localstore"
)
var sourcePeerID = enode.HexID("99d8594b52298567d2ca3f4c441a5ba0140ee9245e26460d01102a52773c73b9")
type mockNetFetcher struct {
peers *sync.Map
sources []*enode.ID
peersPerRequest [][]Address
requestCalled bool
offerCalled bool
quit <-chan struct{}
ctx context.Context
hopCounts []uint8
mu sync.Mutex
}
func (m *mockNetFetcher) Offer(source *enode.ID) {
m.offerCalled = true
m.sources = append(m.sources, source)
}
func (m *mockNetFetcher) Request(hopCount uint8) {
m.mu.Lock()
defer m.mu.Unlock()
m.requestCalled = true
var peers []Address
m.peers.Range(func(key interface{}, _ interface{}) bool {
peers = append(peers, common.FromHex(key.(string)))
return true
})
m.peersPerRequest = append(m.peersPerRequest, peers)
m.hopCounts = append(m.hopCounts, hopCount)
}
type mockNetFetchFuncFactory struct {
fetcher *mockNetFetcher
}
func (m *mockNetFetchFuncFactory) newMockNetFetcher(ctx context.Context, _ Address, peers *sync.Map) NetFetcher {
m.fetcher.peers = peers
m.fetcher.quit = ctx.Done()
m.fetcher.ctx = ctx
return m.fetcher
}
func newTestNetStore(t *testing.T) (netStore *NetStore, fetcher *mockNetFetcher, cleanup func()) {
t.Helper()
dir, err := ioutil.TempDir("", "swarm-storage-")
if err != nil {
t.Fatal(err)
}
localStore, err := localstore.New(dir, make([]byte, 32), nil)
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
}
cleanup = func() {
localStore.Close()
os.RemoveAll(dir)
}
fetcher = new(mockNetFetcher)
mockNetFetchFuncFactory := &mockNetFetchFuncFactory{
fetcher: fetcher,
}
netStore, err = NewNetStore(localStore, mockNetFetchFuncFactory.newMockNetFetcher)
if err != nil {
cleanup()
t.Fatal(err)
}
return netStore, fetcher, cleanup
}
// TestNetStoreGetAndPut tests calling NetStore.Get which is blocked until the same chunk is Put.
// After the Put there should no active fetchers, and the context created for the fetcher should
// be cancelled.
func TestNetStoreGetAndPut(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
c := make(chan struct{}) // this channel ensures that the gouroutine with the Put does not run earlier than the Get
putErrC := make(chan error)
go func() {
<-c // wait for the Get to be called
time.Sleep(200 * time.Millisecond) // and a little more so it is surely called
// check if netStore created a fetcher in the Get call for the unavailable chunk
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
putErrC <- errors.New("Expected netStore to use a fetcher for the Get call")
return
}
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
if err != nil {
putErrC <- fmt.Errorf("Expected no err got %v", err)
return
}
putErrC <- nil
}()
close(c)
recChunk, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address()) // this is blocked until the Put above is done
if err != nil {
t.Fatalf("Expected no err got %v", err)
}
if err := <-putErrC; err != nil {
t.Fatal(err)
}
// the retrieved chunk should be the same as what we Put
if !bytes.Equal(recChunk.Address(), ch.Address()) || !bytes.Equal(recChunk.Data(), ch.Data()) {
t.Fatalf("Different chunk received than what was put")
}
// the chunk is already available locally, so there should be no active fetchers waiting for it
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after delivery")
}
// A fetcher was created when the Get was called (and the chunk was not available). The chunk
// was delivered with the Put call, so the fetcher should be cancelled now.
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
// TestNetStoreGetAndPut tests calling NetStore.Put and then NetStore.Get.
// After the Put the chunk is available locally, so the Get can just retrieve it from LocalStore,
// there is no need to create fetchers.
func TestNetStoreGetAfterPut(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
// First we Put the chunk, so the chunk will be available locally
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
if err != nil {
t.Fatalf("Expected no err got %v", err)
}
// Get should retrieve the chunk from LocalStore, without creating fetcher
recChunk, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
if err != nil {
t.Fatalf("Expected no err got %v", err)
}
// the retrieved chunk should be the same as what we Put
if !bytes.Equal(recChunk.Address(), ch.Address()) || !bytes.Equal(recChunk.Data(), ch.Data()) {
t.Fatalf("Different chunk received than what was put")
}
// no fetcher offer or request should be created for a locally available chunk
if fetcher.offerCalled || fetcher.requestCalled {
t.Fatal("NetFetcher.offerCalled or requestCalled not expected to be called")
}
// no fetchers should be created for a locally available chunk
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to not have fetcher")
}
}
// TestNetStoreGetTimeout tests a Get call for an unavailable chunk and waits for timeout
func TestNetStoreGetTimeout(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
c := make(chan struct{}) // this channel ensures that the gouroutine does not run earlier than the Get
fetcherErrC := make(chan error)
go func() {
<-c // wait for the Get to be called
time.Sleep(200 * time.Millisecond) // and a little more so it is surely called
// check if netStore created a fetcher in the Get call for the unavailable chunk
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
fetcherErrC <- errors.New("Expected netStore to use a fetcher for the Get call")
return
}
fetcherErrC <- nil
}()
close(c)
// We call Get on this chunk, which is not in LocalStore. We don't Put it at all, so there will
// be a timeout
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
// Check if the timeout happened
if err != context.DeadlineExceeded {
t.Fatalf("Expected context.DeadLineExceeded err got %v", err)
}
if err := <-fetcherErrC; err != nil {
t.Fatal(err)
}
// A fetcher was created, check if it has been removed after timeout
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after timeout")
}
// Check if the fetcher context has been cancelled after the timeout
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
// TestNetStoreGetCancel tests a Get call for an unavailable chunk, then cancels the context and checks
// the errors
func TestNetStoreGetCancel(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
c := make(chan struct{}) // this channel ensures that the gouroutine with the cancel does not run earlier than the Get
fetcherErrC := make(chan error, 1)
go func() {
<-c // wait for the Get to be called
time.Sleep(200 * time.Millisecond) // and a little more so it is surely called
// check if netStore created a fetcher in the Get call for the unavailable chunk
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
fetcherErrC <- errors.New("Expected netStore to use a fetcher for the Get call")
return
}
fetcherErrC <- nil
cancel()
}()
close(c)
// We call Get with an unavailable chunk, so it will create a fetcher and wait for delivery
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
if err := <-fetcherErrC; err != nil {
t.Fatal(err)
}
// After the context is cancelled above Get should return with an error
if err != context.Canceled {
t.Fatalf("Expected context.Canceled err got %v", err)
}
// A fetcher was created, check if it has been removed after cancel
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after cancel")
}
// Check if the fetcher context has been cancelled after the request context cancel
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
// TestNetStoreMultipleGetAndPut tests four Get calls for the same unavailable chunk. The chunk is
// delivered with a Put, we have to make sure all Get calls return, and they use a single fetcher
// for the chunk retrieval
func TestNetStoreMultipleGetAndPut(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
putErrC := make(chan error)
go func() {
// sleep to make sure Put is called after all the Get
time.Sleep(500 * time.Millisecond)
// check if netStore created exactly one fetcher for all Get calls
if netStore.fetchers.Len() != 1 {
putErrC <- errors.New("Expected netStore to use one fetcher for all Get calls")
return
}
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
if err != nil {
putErrC <- fmt.Errorf("Expected no err got %v", err)
return
}
putErrC <- nil
}()
count := 4
// call Get 4 times for the same unavailable chunk. The calls will be blocked until the Put above.
errC := make(chan error)
for i := 0; i < count; i++ {
go func() {
recChunk, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
if err != nil {
errC <- fmt.Errorf("Expected no err got %v", err)
}
if !bytes.Equal(recChunk.Address(), ch.Address()) || !bytes.Equal(recChunk.Data(), ch.Data()) {
errC <- errors.New("Different chunk received than what was put")
}
errC <- nil
}()
}
if err := <-putErrC; err != nil {
t.Fatal(err)
}
timeout := time.After(1 * time.Second)
// The Get calls should return after Put, so no timeout expected
for i := 0; i < count; i++ {
select {
case err := <-errC:
if err != nil {
t.Fatal(err)
}
case <-timeout:
t.Fatalf("Timeout waiting for Get calls to return")
}
}
// A fetcher was created, check if it has been removed after cancel
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after delivery")
}
// A fetcher was created, check if it has been removed after delivery
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
// TestNetStoreFetchFuncTimeout tests a FetchFunc call for an unavailable chunk and waits for timeout
func TestNetStoreFetchFuncTimeout(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
// FetchFunc is called for an unavaible chunk, so the returned wait function should not be nil
wait := netStore.FetchFunc(ctx, chunk.Address())
if wait == nil {
t.Fatal("Expected wait function to be not nil")
}
// There should an active fetcher for the chunk after the FetchFunc call
if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil {
t.Fatalf("Expected netStore to have one fetcher for the requested chunk")
}
// wait function should timeout because we don't deliver the chunk with a Put
err := wait(ctx)
if err != context.DeadlineExceeded {
t.Fatalf("Expected context.DeadLineExceeded err got %v", err)
}
// the fetcher should be removed after timeout
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after timeout")
}
// the fetcher context should be cancelled after timeout
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
// TestNetStoreFetchFuncAfterPut tests that the FetchFunc should return nil for a locally available chunk
func TestNetStoreFetchFuncAfterPut(t *testing.T) {
netStore, _, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
// We deliver the created the chunk with a Put
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
if err != nil {
t.Fatalf("Expected no err got %v", err)
}
// FetchFunc should return nil, because the chunk is available locally, no need to fetch it
wait := netStore.FetchFunc(ctx, ch.Address())
if wait != nil {
t.Fatal("Expected wait to be nil")
}
// No fetchers should be created at all
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to not have fetcher")
}
}
// TestNetStoreGetCallsRequest tests if Get created a request on the NetFetcher for an unavailable chunk
func TestNetStoreGetCallsRequest(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx := context.WithValue(context.Background(), "hopcount", uint8(5))
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
// We call get for a not available chunk, it will timeout because the chunk is not delivered
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
if err != context.DeadlineExceeded {
t.Fatalf("Expected context.DeadlineExceeded err got %v", err)
}
// NetStore should call NetFetcher.Request and wait for the chunk
if !fetcher.requestCalled {
t.Fatal("Expected NetFetcher.Request to be called")
}
if fetcher.hopCounts[0] != 5 {
t.Fatalf("Expected NetFetcher.Request be called with hopCount 5, got %v", fetcher.hopCounts[0])
}
}
// TestNetStoreGetCallsOffer tests if Get created a request on the NetFetcher for an unavailable chunk
// in case of a source peer provided in the context.
func TestNetStoreGetCallsOffer(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
// If a source peer is added to the context, NetStore will handle it as an offer
ctx := context.WithValue(context.Background(), "source", sourcePeerID.String())
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
// We call get for a not available chunk, it will timeout because the chunk is not delivered
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
if err != context.DeadlineExceeded {
t.Fatalf("Expect error %v got %v", context.DeadlineExceeded, err)
}
// NetStore should call NetFetcher.Offer with the source peer
if !fetcher.offerCalled {
t.Fatal("Expected NetFetcher.Request to be called")
}
if len(fetcher.sources) != 1 {
t.Fatalf("Expected fetcher sources length 1 got %v", len(fetcher.sources))
}
if fetcher.sources[0].String() != sourcePeerID.String() {
t.Fatalf("Expected fetcher source %v got %v", sourcePeerID, fetcher.sources[0])
}
}
// TestNetStoreFetcherCountPeers tests multiple NetStore.Get calls with peer in the context.
// There is no Put call, so the Get calls timeout
func TestNetStoreFetcherCountPeers(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
addr := randomAddr()
peers := []string{randomAddr().Hex(), randomAddr().Hex(), randomAddr().Hex()}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
errC := make(chan error)
nrGets := 3
// Call Get 3 times with a peer in context
for i := 0; i < nrGets; i++ {
peer := peers[i]
go func() {
ctx := context.WithValue(ctx, "peer", peer)
_, err := netStore.Get(ctx, chunk.ModeGetRequest, addr)
errC <- err
}()
}
// All 3 Get calls should timeout
for i := 0; i < nrGets; i++ {
err := <-errC
if err != context.DeadlineExceeded {
t.Fatalf("Expected \"%v\" error got \"%v\"", context.DeadlineExceeded, err)
}
}
// fetcher should be closed after timeout
select {
case <-fetcher.quit:
case <-time.After(3 * time.Second):
t.Fatalf("mockNetFetcher not closed after timeout")
}
// All 3 peers should be given to NetFetcher after the 3 Get calls
if len(fetcher.peersPerRequest) != nrGets {
t.Fatalf("Expected 3 got %v", len(fetcher.peersPerRequest))
}
for i, peers := range fetcher.peersPerRequest {
if len(peers) < i+1 {
t.Fatalf("Expected at least %v got %v", i+1, len(peers))
}
}
}
// TestNetStoreFetchFuncCalledMultipleTimes calls the wait function given by FetchFunc three times,
// and checks there is still exactly one fetcher for one chunk. Afthe chunk is delivered, it checks
// if the fetcher is closed.
func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
ch := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
// FetchFunc should return a non-nil wait function, because the chunk is not available
wait := netStore.FetchFunc(ctx, ch.Address())
if wait == nil {
t.Fatal("Expected wait function to be not nil")
}
// There should be exactly one fetcher for the chunk
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
t.Fatalf("Expected netStore to have one fetcher for the requested chunk")
}
// Call wait three times in parallel
count := 3
errC := make(chan error)
for i := 0; i < count; i++ {
go func() {
errC <- wait(ctx)
}()
}
// sleep a little so the wait functions are called above
time.Sleep(100 * time.Millisecond)
// there should be still only one fetcher, because all wait calls are for the same chunk
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
t.Fatal("Expected netStore to have one fetcher for the requested chunk")
}
// Deliver the chunk with a Put
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
if err != nil {
t.Fatalf("Expected no err got %v", err)
}
// wait until all wait calls return (because the chunk is delivered)
for i := 0; i < count; i++ {
err := <-errC
if err != nil {
t.Fatal(err)
}
}
// There should be no more fetchers for the delivered chunk
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after delivery")
}
// The context for the fetcher should be cancelled after delivery
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
// TestNetStoreFetcherLifeCycleWithTimeout is similar to TestNetStoreFetchFuncCalledMultipleTimes,
// the only difference is that we don't deilver the chunk, just wait for timeout
func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) {
netStore, fetcher, cleanup := newTestNetStore(t)
defer cleanup()
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
// FetchFunc should return a non-nil wait function, because the chunk is not available
wait := netStore.FetchFunc(ctx, chunk.Address())
if wait == nil {
t.Fatal("Expected wait function to be not nil")
}
// There should be exactly one fetcher for the chunk
if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil {
t.Fatalf("Expected netStore to have one fetcher for the requested chunk")
}
// Call wait three times in parallel
count := 3
errC := make(chan error)
for i := 0; i < count; i++ {
go func() {
rctx, rcancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer rcancel()
err := wait(rctx)
if err != context.DeadlineExceeded {
errC <- fmt.Errorf("Expected err %v got %v", context.DeadlineExceeded, err)
return
}
errC <- nil
}()
}
// wait until all wait calls timeout
for i := 0; i < count; i++ {
err := <-errC
if err != nil {
t.Fatal(err)
}
}
// There should be no more fetchers after timeout
if netStore.fetchers.Len() != 0 {
t.Fatal("Expected netStore to remove the fetcher after delivery")
}
// The context for the fetcher should be cancelled after timeout
select {
case <-fetcher.ctx.Done():
default:
t.Fatal("Expected fetcher context to be cancelled")
}
}
func randomAddr() Address {
addr := make([]byte, 32)
rand.Read(addr)
return Address(addr)
}

58
storage/request.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
import (
"sync"
"time"
"github.com/ethersphere/swarm/network/timeouts"
"github.com/ethereum/go-ethereum/p2p/enode"
)
// Request encapsulates all the necessary arguments when making a request to NetStore.
// These could have also been added as part of the interface of NetStore.Get, but a request struct seemed
// like a better option
type Request struct {
Addr Address // chunk address
Origin enode.ID // who is sending us that request? we compare Origin to the suggested peer from RequestFromPeers
PeersToSkip sync.Map // peers not to request chunk from
}
// NewRequest returns a new instance of Request based on chunk address skip check and
// a map of peers to skip.
func NewRequest(addr Address) *Request {
return &Request{
Addr: addr,
}
}
// SkipPeer returns if the peer with nodeID should not be requested to deliver a chunk.
// Peers to skip are kept per Request and for a time period of FailedPeerSkipDelay.
func (r *Request) SkipPeer(nodeID string) bool {
val, ok := r.PeersToSkip.Load(nodeID)
if !ok {
return false
}
t, ok := val.(time.Time)
if ok && time.Now().After(t.Add(timeouts.FailedPeerSkipDelay)) {
r.PeersToSkip.Delete(nodeID)
return false
}
return true
}