simplification of Fetchers (#1344)
This commit is contained in:
committed by
GitHub
parent
21adbe2fac
commit
0b724bd4d5
@ -192,12 +192,13 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error)
|
||||
ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout)
|
||||
defer cancel()
|
||||
|
||||
ch, err := h.chunkStore.Get(ctx, chunk.ModeGetLookup, id.Addr())
|
||||
r := storage.NewRequest(id.Addr())
|
||||
ch, err := h.chunkStore.Get(ctx, chunk.ModeGetLookup, r)
|
||||
if err != nil {
|
||||
if err == context.DeadlineExceeded { // chunk not found
|
||||
if err == context.DeadlineExceeded || err == storage.ErrNoSuitablePeer { // chunk not found
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err //something else happened or context was cancelled.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var request Request
|
||||
|
@ -58,6 +58,8 @@ var TimeAfter = time.After
|
||||
// It should return <nil> if a value is found, but its timestamp is higher than "now"
|
||||
// It should only return an error in case the handler wants to stop the
|
||||
// lookup process entirely.
|
||||
// If the context is canceled, it must return context.Canceled
|
||||
|
||||
type ReadFunc func(ctx context.Context, epoch Epoch, now uint64) (interface{}, error)
|
||||
|
||||
// NoClue is a hint that can be provided when the Lookup caller does not have
|
||||
|
@ -18,8 +18,8 @@ package feed
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/p2p/enode"
|
||||
"github.com/ethersphere/swarm/chunk"
|
||||
@ -39,17 +39,6 @@ func (t *TestHandler) Close() {
|
||||
t.chunkStore.Close()
|
||||
}
|
||||
|
||||
type mockNetFetcher struct{}
|
||||
|
||||
func (m *mockNetFetcher) Request(hopCount uint8) {
|
||||
}
|
||||
func (m *mockNetFetcher) Offer(source *enode.ID) {
|
||||
}
|
||||
|
||||
func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetFetcher {
|
||||
return &mockNetFetcher{}
|
||||
}
|
||||
|
||||
// NewTestHandler creates Handler object to be used for testing purposes.
|
||||
func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) {
|
||||
path := filepath.Join(datadir, testDbDirName)
|
||||
@ -62,11 +51,10 @@ func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error)
|
||||
|
||||
localStore := chunk.NewValidatorStore(db, storage.NewContentAddressValidator(storage.MakeHashFunc(feedsHashAlgorithm)), fh)
|
||||
|
||||
netStore, err := storage.NewNetStore(localStore, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
netStore := storage.NewNetStore(localStore, enode.ID{})
|
||||
netStore.RemoteGet = func(ctx context.Context, req *storage.Request, localID enode.ID) (*enode.ID, error) {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
netStore.NewNetFetcherFunc = newFakeNetFetcher
|
||||
fh.SetStore(netStore)
|
||||
return &TestHandler{fh}, nil
|
||||
}
|
||||
|
@ -39,9 +39,8 @@ implementation for storage or retrieval.
|
||||
*/
|
||||
|
||||
const (
|
||||
defaultLDBCapacity = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB
|
||||
defaultCacheCapacity = 10000 // capacity for in-memory chunks' cache
|
||||
defaultChunkRequestsCacheCapacity = 5000000 // capacity for container holding outgoing requests for chunks. should be set to LevelDB capacity
|
||||
defaultLDBCapacity = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB
|
||||
defaultCacheCapacity = 10000 // capacity for in-memory chunks' cache
|
||||
)
|
||||
|
||||
type FileStore struct {
|
||||
|
46
storage/lnetstore.go
Normal file
46
storage/lnetstore.go
Normal file
@ -0,0 +1,46 @@
|
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/ethersphere/swarm/chunk"
|
||||
"github.com/ethersphere/swarm/network/timeouts"
|
||||
)
|
||||
|
||||
// LNetStore is a wrapper of NetStore, which implements the chunk.Store interface. It is used only by the FileStore,
|
||||
// the component used by the Swarm API to store and retrieve content and to split and join chunks.
|
||||
type LNetStore struct {
|
||||
*NetStore
|
||||
}
|
||||
|
||||
// NewLNetStore is a constructor for LNetStore
|
||||
func NewLNetStore(store *NetStore) *LNetStore {
|
||||
return &LNetStore{
|
||||
NetStore: store,
|
||||
}
|
||||
}
|
||||
|
||||
// Get converts a chunk reference to a chunk Request (with empty Origin), handled by the NetStore, and
|
||||
// returns the requested chunk, or error.
|
||||
func (n *LNetStore) Get(ctx context.Context, mode chunk.ModeGet, ref Address) (ch Chunk, err error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeouts.FetcherGlobalTimeout)
|
||||
defer cancel()
|
||||
|
||||
return n.NetStore.Get(ctx, mode, NewRequest(ref))
|
||||
}
|
@ -18,318 +18,291 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/p2p/enode"
|
||||
"github.com/ethersphere/swarm/chunk"
|
||||
"github.com/ethersphere/swarm/log"
|
||||
"github.com/ethersphere/swarm/network/timeouts"
|
||||
"github.com/ethersphere/swarm/spancontext"
|
||||
"github.com/opentracing/opentracing-go"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethereum/go-ethereum/p2p/enode"
|
||||
olog "github.com/opentracing/opentracing-go/log"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
type (
|
||||
NewNetFetcherFunc func(ctx context.Context, addr Address, peers *sync.Map) NetFetcher
|
||||
const (
|
||||
// capacity for the fetchers LRU cache
|
||||
fetchersCapacity = 500000
|
||||
)
|
||||
|
||||
type NetFetcher interface {
|
||||
Request(hopCount uint8)
|
||||
Offer(source *enode.ID)
|
||||
var (
|
||||
ErrNoSuitablePeer = errors.New("no suitable peer")
|
||||
)
|
||||
|
||||
// Fetcher is a struct which maintains state of remote requests.
|
||||
// Fetchers are stored in fetchers map and signal to all interested parties if a given chunk is delivered
|
||||
// the mutex controls who closes the channel, and make sure we close the channel only once
|
||||
type Fetcher struct {
|
||||
Delivered chan struct{} // when closed, it means that the chunk this Fetcher refers to is delivered
|
||||
|
||||
// it is possible for multiple actors to be delivering the same chunk,
|
||||
// for example through syncing and through retrieve request. however we want the `Delivered` channel to be closed only
|
||||
// once, even if we put the same chunk multiple times in the NetStore.
|
||||
once sync.Once
|
||||
|
||||
CreatedAt time.Time // timestamp when the fetcher was created, used for metrics measuring lifetime of fetchers
|
||||
CreatedBy string // who created the fetcher - "request" or "syncing", used for metrics measuring lifecycle of fetchers
|
||||
|
||||
RequestedBySyncer bool // whether we have issued at least once a request through Offered/Wanted hashes flow
|
||||
}
|
||||
|
||||
// NetStore is an extension of local storage
|
||||
// NewFetcher is a constructor for a Fetcher
|
||||
func NewFetcher() *Fetcher {
|
||||
return &Fetcher{make(chan struct{}), sync.Once{}, time.Now(), "", false}
|
||||
}
|
||||
|
||||
// SafeClose signals to interested parties (those waiting for a signal on fi.Delivered) that a chunk is delivered.
|
||||
// It closes the fi.Delivered channel through the sync.Once object, because it is possible for a chunk to be
|
||||
// delivered multiple times concurrently.
|
||||
func (fi *Fetcher) SafeClose() {
|
||||
fi.once.Do(func() {
|
||||
close(fi.Delivered)
|
||||
})
|
||||
}
|
||||
|
||||
type RemoteGetFunc func(ctx context.Context, req *Request, localID enode.ID) (*enode.ID, error)
|
||||
|
||||
// NetStore is an extension of LocalStore
|
||||
// it implements the ChunkStore interface
|
||||
// on request it initiates remote cloud retrieval using a fetcher
|
||||
// fetchers are unique to a chunk and are stored in fetchers LRU memory cache
|
||||
// fetchFuncFactory is a factory object to create a fetch function for a specific chunk address
|
||||
// on request it initiates remote cloud retrieval
|
||||
type NetStore struct {
|
||||
chunk.Store
|
||||
mu sync.Mutex
|
||||
fetchers *lru.Cache
|
||||
NewNetFetcherFunc NewNetFetcherFunc
|
||||
closeC chan struct{}
|
||||
localID enode.ID // our local enode - used when issuing RetrieveRequests
|
||||
fetchers *lru.Cache
|
||||
putMu sync.Mutex
|
||||
requestGroup singleflight.Group
|
||||
RemoteGet RemoteGetFunc
|
||||
}
|
||||
|
||||
var fetcherTimeout = 2 * time.Minute // timeout to cancel the fetcher even if requests are coming in
|
||||
// NewNetStore creates a new NetStore using the provided chunk.Store and localID of the node.
|
||||
func NewNetStore(store chunk.Store, localID enode.ID) *NetStore {
|
||||
fetchers, _ := lru.New(fetchersCapacity)
|
||||
|
||||
// NewNetStore creates a new NetStore object using the given local store. newFetchFunc is a
|
||||
// constructor function that can create a fetch function for a specific chunk address.
|
||||
func NewNetStore(store chunk.Store, nnf NewNetFetcherFunc) (*NetStore, error) {
|
||||
fetchers, err := lru.New(defaultChunkRequestsCacheCapacity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &NetStore{
|
||||
Store: store,
|
||||
fetchers: fetchers,
|
||||
NewNetFetcherFunc: nnf,
|
||||
closeC: make(chan struct{}),
|
||||
}, nil
|
||||
fetchers: fetchers,
|
||||
Store: store,
|
||||
localID: localID,
|
||||
}
|
||||
}
|
||||
|
||||
// Put stores a chunk in localstore, and delivers to all requestor peers using the fetcher stored in
|
||||
// the fetchers cache
|
||||
func (n *NetStore) Put(ctx context.Context, mode chunk.ModePut, ch Chunk) (bool, error) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
n.putMu.Lock()
|
||||
defer n.putMu.Unlock()
|
||||
|
||||
// put to the chunk to the store, there should be no error
|
||||
log.Trace("netstore.put", "ref", ch.Address().String(), "mode", mode)
|
||||
|
||||
// put the chunk to the localstore, there should be no error
|
||||
exists, err := n.Store.Put(ctx, mode, ch)
|
||||
if err != nil {
|
||||
return exists, err
|
||||
}
|
||||
|
||||
// if chunk is now put in the store, check if there was an active fetcher and call deliver on it
|
||||
// (this delivers the chunk to requestors via the fetcher)
|
||||
log.Trace("n.getFetcher", "ref", ch.Address())
|
||||
if f := n.getFetcher(ch.Address()); f != nil {
|
||||
log.Trace("n.getFetcher deliver", "ref", ch.Address())
|
||||
f.deliver(ctx, ch)
|
||||
// notify RemoteGet (or SwarmSyncerClient) about a chunk delivery and it being stored
|
||||
fi, ok := n.fetchers.Get(ch.Address().String())
|
||||
if ok {
|
||||
// we need SafeClose, because it is possible for a chunk to both be
|
||||
// delivered through syncing and through a retrieve request
|
||||
fii := fi.(*Fetcher)
|
||||
fii.SafeClose()
|
||||
log.Trace("netstore.put chunk delivered and stored", "ref", ch.Address().String())
|
||||
|
||||
metrics.GetOrRegisterResettingTimer(fmt.Sprintf("netstore.fetcher.lifetime.%s", fii.CreatedBy), nil).UpdateSince(fii.CreatedAt)
|
||||
|
||||
// helper snippet to log if a chunk took way to long to be delivered
|
||||
slowChunkDeliveryThreshold := 5 * time.Second
|
||||
if time.Since(fii.CreatedAt) > slowChunkDeliveryThreshold {
|
||||
log.Trace("netstore.put slow chunk delivery", "ref", ch.Address().String())
|
||||
}
|
||||
|
||||
n.fetchers.Remove(ch.Address().String())
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// Get retrieves the chunk from the NetStore DPA synchronously.
|
||||
// It calls NetStore.get, and if the chunk is not in local Storage
|
||||
// it calls fetch with the request, which blocks until the chunk
|
||||
// arrived or context is done
|
||||
func (n *NetStore) Get(rctx context.Context, mode chunk.ModeGet, ref Address) (Chunk, error) {
|
||||
chunk, fetch, err := n.get(rctx, mode, ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chunk != nil {
|
||||
// this is not measuring how long it takes to get the chunk for the localstore, but
|
||||
// rather just adding a span for clarity when inspecting traces in Jaeger, in order
|
||||
// to make it easier to reason which is the node that actually delivered a chunk.
|
||||
_, sp := spancontext.StartSpan(
|
||||
rctx,
|
||||
"localstore.get")
|
||||
defer sp.Finish()
|
||||
|
||||
return chunk, nil
|
||||
}
|
||||
return fetch(rctx)
|
||||
}
|
||||
|
||||
// FetchFunc returns nil if the store contains the given address. Otherwise it returns a wait function,
|
||||
// which returns after the chunk is available or the context is done
|
||||
func (n *NetStore) FetchFunc(ctx context.Context, ref Address) func(context.Context) error {
|
||||
chunk, fetch, _ := n.get(ctx, chunk.ModeGetRequest, ref)
|
||||
if chunk != nil {
|
||||
return nil
|
||||
}
|
||||
return func(ctx context.Context) error {
|
||||
_, err := fetch(ctx)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Close chunk store
|
||||
func (n *NetStore) Close() (err error) {
|
||||
close(n.closeC)
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
for _, key := range n.fetchers.Keys() {
|
||||
if f, ok := n.fetchers.Get(key); ok {
|
||||
if fetch, ok := f.(*fetcher); ok {
|
||||
wg.Add(1)
|
||||
go func(fetch *fetcher) {
|
||||
defer wg.Done()
|
||||
fetch.cancel()
|
||||
|
||||
select {
|
||||
case <-fetch.deliveredC:
|
||||
case <-fetch.cancelledC:
|
||||
}
|
||||
}(fetch)
|
||||
}
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
func (n *NetStore) Close() error {
|
||||
return n.Store.Close()
|
||||
}
|
||||
|
||||
// get attempts at retrieving the chunk from LocalStore
|
||||
// If it is not found then using getOrCreateFetcher:
|
||||
// 1. Either there is already a fetcher to retrieve it
|
||||
// 2. A new fetcher is created and saved in the fetchers cache
|
||||
// From here on, all Get will hit on this fetcher until the chunk is delivered
|
||||
// or all fetcher contexts are done.
|
||||
// It returns a chunk, a fetcher function and an error
|
||||
// If chunk is nil, the returned fetch function needs to be called with a context to return the chunk.
|
||||
func (n *NetStore) get(ctx context.Context, mode chunk.ModeGet, ref Address) (Chunk, func(context.Context) (Chunk, error), error) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
// Get retrieves a chunk
|
||||
// If it is not found in the LocalStore then it uses RemoteGet to fetch from the network.
|
||||
func (n *NetStore) Get(ctx context.Context, mode chunk.ModeGet, req *Request) (Chunk, error) {
|
||||
metrics.GetOrRegisterCounter("netstore.get", nil).Inc(1)
|
||||
start := time.Now()
|
||||
|
||||
chunk, err := n.Store.Get(ctx, mode, ref)
|
||||
ref := req.Addr
|
||||
|
||||
log.Trace("netstore.get", "ref", ref.String())
|
||||
|
||||
ch, err := n.Store.Get(ctx, mode, ref)
|
||||
if err != nil {
|
||||
// TODO: Fix comparison - we should be comparing against leveldb.ErrNotFound, this error should be wrapped.
|
||||
// TODO: fix comparison - we should be comparing against leveldb.ErrNotFound, this error should be wrapped.
|
||||
if err != ErrChunkNotFound && err != leveldb.ErrNotFound {
|
||||
log.Debug("Received error from LocalStore other than ErrNotFound", "err", err)
|
||||
log.Error("localstore get error", "err", err)
|
||||
}
|
||||
// The chunk is not available in the LocalStore, let's get the fetcher for it, or create a new one
|
||||
// if it doesn't exist yet
|
||||
f := n.getOrCreateFetcher(ctx, ref)
|
||||
// If the caller needs the chunk, it has to use the returned fetch function to get it
|
||||
return nil, f.Fetch, nil
|
||||
}
|
||||
|
||||
return chunk, nil, nil
|
||||
}
|
||||
log.Trace("netstore.chunk-not-in-localstore", "ref", ref.String())
|
||||
|
||||
// getOrCreateFetcher attempts at retrieving an existing fetchers
|
||||
// if none exists, creates one and saves it in the fetchers cache
|
||||
// caller must hold the lock
|
||||
func (n *NetStore) getOrCreateFetcher(ctx context.Context, ref Address) *fetcher {
|
||||
if f := n.getFetcher(ref); f != nil {
|
||||
return f
|
||||
}
|
||||
v, err, _ := n.requestGroup.Do(ref.String(), func() (interface{}, error) {
|
||||
// currently we issue a retrieve request if a fetcher
|
||||
// has already been created by a syncer for that particular chunk.
|
||||
// so it is possible to
|
||||
// have 2 in-flight requests for the same chunk - one by a
|
||||
// syncer (offered/wanted/deliver flow) and one from
|
||||
// here - retrieve request
|
||||
fi, _, ok := n.GetOrCreateFetcher(ctx, ref, "request")
|
||||
if ok {
|
||||
err := n.RemoteFetch(ctx, req, fi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// no fetcher for the given address, we have to create a new one
|
||||
key := hex.EncodeToString(ref)
|
||||
// create the context during which fetching is kept alive
|
||||
cctx, cancel := context.WithTimeout(ctx, fetcherTimeout)
|
||||
// destroy is called when all requests finish
|
||||
destroy := func() {
|
||||
// remove fetcher from fetchers
|
||||
n.fetchers.Remove(key)
|
||||
// stop fetcher by cancelling context called when
|
||||
// all requests cancelled/timedout or chunk is delivered
|
||||
cancel()
|
||||
}
|
||||
// peers always stores all the peers which have an active request for the chunk. It is shared
|
||||
// between fetcher and the NewFetchFunc function. It is needed by the NewFetchFunc because
|
||||
// the peers which requested the chunk should not be requested to deliver it.
|
||||
peers := &sync.Map{}
|
||||
ch, err := n.Store.Get(ctx, mode, ref)
|
||||
if err != nil {
|
||||
log.Error(err.Error(), "ref", ref)
|
||||
return nil, errors.New("item should have been in localstore, but it is not")
|
||||
}
|
||||
|
||||
cctx, sp := spancontext.StartSpan(
|
||||
cctx,
|
||||
"netstore.fetcher",
|
||||
)
|
||||
// fi could be nil (when ok == false) if the chunk was added to the NetStore between n.store.Get and the call to n.GetOrCreateFetcher
|
||||
if fi != nil {
|
||||
metrics.GetOrRegisterResettingTimer(fmt.Sprintf("fetcher.%s.request", fi.CreatedBy), nil).UpdateSince(start)
|
||||
}
|
||||
|
||||
sp.LogFields(olog.String("ref", ref.String()))
|
||||
fetcher := newFetcher(sp, ref, n.NewNetFetcherFunc(cctx, ref, peers), destroy, peers, n.closeC)
|
||||
n.fetchers.Add(key, fetcher)
|
||||
return ch, nil
|
||||
})
|
||||
|
||||
return fetcher
|
||||
}
|
||||
|
||||
// getFetcher retrieves the fetcher for the given address from the fetchers cache if it exists,
|
||||
// otherwise it returns nil
|
||||
func (n *NetStore) getFetcher(ref Address) *fetcher {
|
||||
key := hex.EncodeToString(ref)
|
||||
f, ok := n.fetchers.Get(key)
|
||||
if ok {
|
||||
return f.(*fetcher)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RequestsCacheLen returns the current number of outgoing requests stored in the cache
|
||||
func (n *NetStore) RequestsCacheLen() int {
|
||||
return n.fetchers.Len()
|
||||
}
|
||||
|
||||
// One fetcher object is responsible to fetch one chunk for one address, and keep track of all the
|
||||
// peers who have requested it and did not receive it yet.
|
||||
type fetcher struct {
|
||||
addr Address // address of chunk
|
||||
chunk Chunk // fetcher can set the chunk on the fetcher
|
||||
deliveredC chan struct{} // chan signalling chunk delivery to requests
|
||||
cancelledC chan struct{} // chan signalling the fetcher has been cancelled (removed from fetchers in NetStore)
|
||||
netFetcher NetFetcher // remote fetch function to be called with a request source taken from the context
|
||||
cancel func() // cleanup function for the remote fetcher to call when all upstream contexts are called
|
||||
peers *sync.Map // the peers which asked for the chunk
|
||||
requestCnt int32 // number of requests on this chunk. If all the requests are done (delivered or context is done) the cancel function is called
|
||||
deliverOnce *sync.Once // guarantees that we only close deliveredC once
|
||||
span opentracing.Span // measure retrieve time per chunk
|
||||
}
|
||||
|
||||
// newFetcher creates a new fetcher object for the fiven addr. fetch is the function which actually
|
||||
// does the retrieval (in non-test cases this is coming from the network package). cancel function is
|
||||
// called either
|
||||
// 1. when the chunk has been fetched all peers have been either notified or their context has been done
|
||||
// 2. the chunk has not been fetched but all context from all the requests has been done
|
||||
// The peers map stores all the peers which have requested chunk.
|
||||
func newFetcher(span opentracing.Span, addr Address, nf NetFetcher, cancel func(), peers *sync.Map, closeC chan struct{}) *fetcher {
|
||||
cancelOnce := &sync.Once{} // cancel should only be called once
|
||||
return &fetcher{
|
||||
addr: addr,
|
||||
deliveredC: make(chan struct{}),
|
||||
deliverOnce: &sync.Once{},
|
||||
cancelledC: closeC,
|
||||
netFetcher: nf,
|
||||
cancel: func() {
|
||||
cancelOnce.Do(func() {
|
||||
cancel()
|
||||
})
|
||||
},
|
||||
peers: peers,
|
||||
span: span,
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch fetches the chunk synchronously, it is called by NetStore.Get is the chunk is not available
|
||||
// locally.
|
||||
func (f *fetcher) Fetch(rctx context.Context) (Chunk, error) {
|
||||
atomic.AddInt32(&f.requestCnt, 1)
|
||||
defer func() {
|
||||
// if all the requests are done the fetcher can be cancelled
|
||||
if atomic.AddInt32(&f.requestCnt, -1) == 0 {
|
||||
f.cancel()
|
||||
}
|
||||
f.span.Finish()
|
||||
}()
|
||||
|
||||
// The peer asking for the chunk. Store in the shared peers map, but delete after the request
|
||||
// has been delivered
|
||||
peer := rctx.Value("peer")
|
||||
if peer != nil {
|
||||
f.peers.Store(peer, time.Now())
|
||||
defer f.peers.Delete(peer)
|
||||
}
|
||||
|
||||
// If there is a source in the context then it is an offer, otherwise a request
|
||||
sourceIF := rctx.Value("source")
|
||||
|
||||
hopCount, _ := rctx.Value("hopcount").(uint8)
|
||||
|
||||
if sourceIF != nil {
|
||||
var source enode.ID
|
||||
if err := source.UnmarshalText([]byte(sourceIF.(string))); err != nil {
|
||||
if err != nil {
|
||||
log.Trace(err.Error(), "ref", ref)
|
||||
return nil, err
|
||||
}
|
||||
f.netFetcher.Offer(&source)
|
||||
|
||||
c := v.(Chunk)
|
||||
|
||||
log.Trace("netstore.singleflight returned", "ref", ref.String(), "err", err)
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
ctx, ssp := spancontext.StartSpan(
|
||||
ctx,
|
||||
"localstore.get")
|
||||
defer ssp.Finish()
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// RemoteFetch is handling the retry mechanism when making a chunk request to our peers.
|
||||
// For a given chunk Request, we call RemoteGet, which selects the next eligible peer and
|
||||
// issues a RetrieveRequest and we wait for a delivery. If a delivery doesn't arrive within the SearchTimeout
|
||||
// we retry.
|
||||
func (n *NetStore) RemoteFetch(ctx context.Context, req *Request, fi *Fetcher) error {
|
||||
// while we haven't timed-out, and while we don't have a chunk,
|
||||
// iterate over peers and try to find a chunk
|
||||
metrics.GetOrRegisterCounter("remote.fetch", nil).Inc(1)
|
||||
|
||||
ref := req.Addr
|
||||
|
||||
for {
|
||||
metrics.GetOrRegisterCounter("remote.fetch.inner", nil).Inc(1)
|
||||
|
||||
ctx, osp := spancontext.StartSpan(
|
||||
ctx,
|
||||
"remote.fetch")
|
||||
osp.LogFields(olog.String("ref", ref.String()))
|
||||
|
||||
log.Trace("remote.fetch", "ref", ref)
|
||||
|
||||
currentPeer, err := n.RemoteGet(ctx, req, n.localID)
|
||||
if err != nil {
|
||||
log.Trace(err.Error(), "ref", ref)
|
||||
osp.LogFields(olog.String("err", err.Error()))
|
||||
osp.Finish()
|
||||
return ErrNoSuitablePeer
|
||||
}
|
||||
|
||||
// add peer to the set of peers to skip from now
|
||||
log.Trace("remote.fetch, adding peer to skip", "ref", ref, "peer", currentPeer.String())
|
||||
req.PeersToSkip.Store(currentPeer.String(), time.Now())
|
||||
|
||||
select {
|
||||
case <-fi.Delivered:
|
||||
log.Trace("remote.fetch, chunk delivered", "ref", ref)
|
||||
|
||||
osp.LogFields(olog.Bool("delivered", true))
|
||||
osp.Finish()
|
||||
return nil
|
||||
case <-time.After(timeouts.SearchTimeout):
|
||||
metrics.GetOrRegisterCounter("remote.fetch.timeout.search", nil).Inc(1)
|
||||
|
||||
osp.LogFields(olog.Bool("timeout", true))
|
||||
osp.Finish()
|
||||
break
|
||||
case <-ctx.Done(): // global fetcher timeout
|
||||
log.Trace("remote.fetch, fail", "ref", ref)
|
||||
metrics.GetOrRegisterCounter("remote.fetch.timeout.global", nil).Inc(1)
|
||||
|
||||
osp.LogFields(olog.Bool("fail", true))
|
||||
osp.Finish()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Has is the storage layer entry point to query the underlying
|
||||
// database to return if it has a chunk or not.
|
||||
func (n *NetStore) Has(ctx context.Context, ref Address) (bool, error) {
|
||||
return n.Store.Has(ctx, ref)
|
||||
}
|
||||
|
||||
// GetOrCreateFetcher returns the Fetcher for a given chunk, if this chunk is not in the LocalStore.
|
||||
// If the chunk is in the LocalStore, it returns nil for the Fetcher and ok == false
|
||||
func (n *NetStore) GetOrCreateFetcher(ctx context.Context, ref Address, interestedParty string) (f *Fetcher, loaded bool, ok bool) {
|
||||
n.putMu.Lock()
|
||||
defer n.putMu.Unlock()
|
||||
|
||||
has, err := n.Store.Has(ctx, ref)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
if has {
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
f = NewFetcher()
|
||||
v, loaded := n.fetchers.Get(ref.String())
|
||||
log.Trace("netstore.has-with-callback.loadorstore", "ref", ref.String(), "loaded", loaded)
|
||||
if loaded {
|
||||
f = v.(*Fetcher)
|
||||
} else {
|
||||
f.netFetcher.Request(hopCount)
|
||||
f.CreatedBy = interestedParty
|
||||
n.fetchers.Add(ref.String(), f)
|
||||
}
|
||||
|
||||
// wait until either the chunk is delivered or the context is done
|
||||
select {
|
||||
case <-rctx.Done():
|
||||
return nil, rctx.Err()
|
||||
case <-f.deliveredC:
|
||||
return f.chunk, nil
|
||||
case <-f.cancelledC:
|
||||
return nil, fmt.Errorf("fetcher cancelled")
|
||||
// if fetcher created by request, but we get a call from syncer, make sure we issue a second request
|
||||
if f.CreatedBy != interestedParty && !f.RequestedBySyncer {
|
||||
f.RequestedBySyncer = true
|
||||
return f, false, true
|
||||
}
|
||||
}
|
||||
|
||||
// deliver is called by NetStore.Put to notify all pending requests
|
||||
func (f *fetcher) deliver(ctx context.Context, ch Chunk) {
|
||||
f.deliverOnce.Do(func() {
|
||||
f.chunk = ch
|
||||
// closing the deliveredC channel will terminate ongoing requests
|
||||
close(f.deliveredC)
|
||||
log.Trace("n.getFetcher close deliveredC", "ref", ch.Address())
|
||||
})
|
||||
return f, loaded, true
|
||||
}
|
||||
|
@ -1,702 +0,0 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/p2p/enode"
|
||||
"github.com/ethersphere/swarm/chunk"
|
||||
"github.com/ethersphere/swarm/storage/localstore"
|
||||
)
|
||||
|
||||
var sourcePeerID = enode.HexID("99d8594b52298567d2ca3f4c441a5ba0140ee9245e26460d01102a52773c73b9")
|
||||
|
||||
type mockNetFetcher struct {
|
||||
peers *sync.Map
|
||||
sources []*enode.ID
|
||||
peersPerRequest [][]Address
|
||||
requestCalled bool
|
||||
offerCalled bool
|
||||
quit <-chan struct{}
|
||||
ctx context.Context
|
||||
hopCounts []uint8
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (m *mockNetFetcher) Offer(source *enode.ID) {
|
||||
m.offerCalled = true
|
||||
m.sources = append(m.sources, source)
|
||||
}
|
||||
|
||||
func (m *mockNetFetcher) Request(hopCount uint8) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.requestCalled = true
|
||||
var peers []Address
|
||||
m.peers.Range(func(key interface{}, _ interface{}) bool {
|
||||
peers = append(peers, common.FromHex(key.(string)))
|
||||
return true
|
||||
})
|
||||
m.peersPerRequest = append(m.peersPerRequest, peers)
|
||||
m.hopCounts = append(m.hopCounts, hopCount)
|
||||
}
|
||||
|
||||
type mockNetFetchFuncFactory struct {
|
||||
fetcher *mockNetFetcher
|
||||
}
|
||||
|
||||
func (m *mockNetFetchFuncFactory) newMockNetFetcher(ctx context.Context, _ Address, peers *sync.Map) NetFetcher {
|
||||
m.fetcher.peers = peers
|
||||
m.fetcher.quit = ctx.Done()
|
||||
m.fetcher.ctx = ctx
|
||||
return m.fetcher
|
||||
}
|
||||
|
||||
func newTestNetStore(t *testing.T) (netStore *NetStore, fetcher *mockNetFetcher, cleanup func()) {
|
||||
t.Helper()
|
||||
|
||||
dir, err := ioutil.TempDir("", "swarm-storage-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
localStore, err := localstore.New(dir, make([]byte, 32), nil)
|
||||
if err != nil {
|
||||
os.RemoveAll(dir)
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleanup = func() {
|
||||
localStore.Close()
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
fetcher = new(mockNetFetcher)
|
||||
mockNetFetchFuncFactory := &mockNetFetchFuncFactory{
|
||||
fetcher: fetcher,
|
||||
}
|
||||
netStore, err = NewNetStore(localStore, mockNetFetchFuncFactory.newMockNetFetcher)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
t.Fatal(err)
|
||||
}
|
||||
return netStore, fetcher, cleanup
|
||||
}
|
||||
|
||||
// TestNetStoreGetAndPut tests calling NetStore.Get which is blocked until the same chunk is Put.
|
||||
// After the Put there should no active fetchers, and the context created for the fetcher should
|
||||
// be cancelled.
|
||||
func TestNetStoreGetAndPut(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c := make(chan struct{}) // this channel ensures that the gouroutine with the Put does not run earlier than the Get
|
||||
putErrC := make(chan error)
|
||||
go func() {
|
||||
<-c // wait for the Get to be called
|
||||
time.Sleep(200 * time.Millisecond) // and a little more so it is surely called
|
||||
|
||||
// check if netStore created a fetcher in the Get call for the unavailable chunk
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
|
||||
putErrC <- errors.New("Expected netStore to use a fetcher for the Get call")
|
||||
return
|
||||
}
|
||||
|
||||
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
|
||||
if err != nil {
|
||||
putErrC <- fmt.Errorf("Expected no err got %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
putErrC <- nil
|
||||
}()
|
||||
|
||||
close(c)
|
||||
recChunk, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address()) // this is blocked until the Put above is done
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no err got %v", err)
|
||||
}
|
||||
|
||||
if err := <-putErrC; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// the retrieved chunk should be the same as what we Put
|
||||
if !bytes.Equal(recChunk.Address(), ch.Address()) || !bytes.Equal(recChunk.Data(), ch.Data()) {
|
||||
t.Fatalf("Different chunk received than what was put")
|
||||
}
|
||||
// the chunk is already available locally, so there should be no active fetchers waiting for it
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after delivery")
|
||||
}
|
||||
|
||||
// A fetcher was created when the Get was called (and the chunk was not available). The chunk
|
||||
// was delivered with the Put call, so the fetcher should be cancelled now.
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestNetStoreGetAndPut tests calling NetStore.Put and then NetStore.Get.
|
||||
// After the Put the chunk is available locally, so the Get can just retrieve it from LocalStore,
|
||||
// there is no need to create fetchers.
|
||||
func TestNetStoreGetAfterPut(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
// First we Put the chunk, so the chunk will be available locally
|
||||
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no err got %v", err)
|
||||
}
|
||||
|
||||
// Get should retrieve the chunk from LocalStore, without creating fetcher
|
||||
recChunk, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no err got %v", err)
|
||||
}
|
||||
// the retrieved chunk should be the same as what we Put
|
||||
if !bytes.Equal(recChunk.Address(), ch.Address()) || !bytes.Equal(recChunk.Data(), ch.Data()) {
|
||||
t.Fatalf("Different chunk received than what was put")
|
||||
}
|
||||
// no fetcher offer or request should be created for a locally available chunk
|
||||
if fetcher.offerCalled || fetcher.requestCalled {
|
||||
t.Fatal("NetFetcher.offerCalled or requestCalled not expected to be called")
|
||||
}
|
||||
// no fetchers should be created for a locally available chunk
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to not have fetcher")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestNetStoreGetTimeout tests a Get call for an unavailable chunk and waits for timeout
|
||||
func TestNetStoreGetTimeout(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
c := make(chan struct{}) // this channel ensures that the gouroutine does not run earlier than the Get
|
||||
fetcherErrC := make(chan error)
|
||||
go func() {
|
||||
<-c // wait for the Get to be called
|
||||
time.Sleep(200 * time.Millisecond) // and a little more so it is surely called
|
||||
|
||||
// check if netStore created a fetcher in the Get call for the unavailable chunk
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
|
||||
fetcherErrC <- errors.New("Expected netStore to use a fetcher for the Get call")
|
||||
return
|
||||
}
|
||||
|
||||
fetcherErrC <- nil
|
||||
}()
|
||||
|
||||
close(c)
|
||||
// We call Get on this chunk, which is not in LocalStore. We don't Put it at all, so there will
|
||||
// be a timeout
|
||||
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
|
||||
|
||||
// Check if the timeout happened
|
||||
if err != context.DeadlineExceeded {
|
||||
t.Fatalf("Expected context.DeadLineExceeded err got %v", err)
|
||||
}
|
||||
|
||||
if err := <-fetcherErrC; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A fetcher was created, check if it has been removed after timeout
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after timeout")
|
||||
}
|
||||
|
||||
// Check if the fetcher context has been cancelled after the timeout
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreGetCancel tests a Get call for an unavailable chunk, then cancels the context and checks
|
||||
// the errors
|
||||
func TestNetStoreGetCancel(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
|
||||
c := make(chan struct{}) // this channel ensures that the gouroutine with the cancel does not run earlier than the Get
|
||||
fetcherErrC := make(chan error, 1)
|
||||
go func() {
|
||||
<-c // wait for the Get to be called
|
||||
time.Sleep(200 * time.Millisecond) // and a little more so it is surely called
|
||||
// check if netStore created a fetcher in the Get call for the unavailable chunk
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
|
||||
fetcherErrC <- errors.New("Expected netStore to use a fetcher for the Get call")
|
||||
return
|
||||
}
|
||||
|
||||
fetcherErrC <- nil
|
||||
cancel()
|
||||
}()
|
||||
|
||||
close(c)
|
||||
|
||||
// We call Get with an unavailable chunk, so it will create a fetcher and wait for delivery
|
||||
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
|
||||
|
||||
if err := <-fetcherErrC; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// After the context is cancelled above Get should return with an error
|
||||
if err != context.Canceled {
|
||||
t.Fatalf("Expected context.Canceled err got %v", err)
|
||||
}
|
||||
|
||||
// A fetcher was created, check if it has been removed after cancel
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after cancel")
|
||||
}
|
||||
|
||||
// Check if the fetcher context has been cancelled after the request context cancel
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreMultipleGetAndPut tests four Get calls for the same unavailable chunk. The chunk is
|
||||
// delivered with a Put, we have to make sure all Get calls return, and they use a single fetcher
|
||||
// for the chunk retrieval
|
||||
func TestNetStoreMultipleGetAndPut(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
putErrC := make(chan error)
|
||||
go func() {
|
||||
// sleep to make sure Put is called after all the Get
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
// check if netStore created exactly one fetcher for all Get calls
|
||||
if netStore.fetchers.Len() != 1 {
|
||||
putErrC <- errors.New("Expected netStore to use one fetcher for all Get calls")
|
||||
return
|
||||
}
|
||||
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
|
||||
if err != nil {
|
||||
putErrC <- fmt.Errorf("Expected no err got %v", err)
|
||||
return
|
||||
}
|
||||
putErrC <- nil
|
||||
}()
|
||||
|
||||
count := 4
|
||||
// call Get 4 times for the same unavailable chunk. The calls will be blocked until the Put above.
|
||||
errC := make(chan error)
|
||||
for i := 0; i < count; i++ {
|
||||
go func() {
|
||||
recChunk, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
|
||||
if err != nil {
|
||||
errC <- fmt.Errorf("Expected no err got %v", err)
|
||||
}
|
||||
if !bytes.Equal(recChunk.Address(), ch.Address()) || !bytes.Equal(recChunk.Data(), ch.Data()) {
|
||||
errC <- errors.New("Different chunk received than what was put")
|
||||
}
|
||||
errC <- nil
|
||||
}()
|
||||
}
|
||||
|
||||
if err := <-putErrC; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
timeout := time.After(1 * time.Second)
|
||||
|
||||
// The Get calls should return after Put, so no timeout expected
|
||||
for i := 0; i < count; i++ {
|
||||
select {
|
||||
case err := <-errC:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-timeout:
|
||||
t.Fatalf("Timeout waiting for Get calls to return")
|
||||
}
|
||||
}
|
||||
|
||||
// A fetcher was created, check if it has been removed after cancel
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after delivery")
|
||||
}
|
||||
|
||||
// A fetcher was created, check if it has been removed after delivery
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestNetStoreFetchFuncTimeout tests a FetchFunc call for an unavailable chunk and waits for timeout
|
||||
func TestNetStoreFetchFuncTimeout(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
chunk := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
// FetchFunc is called for an unavaible chunk, so the returned wait function should not be nil
|
||||
wait := netStore.FetchFunc(ctx, chunk.Address())
|
||||
if wait == nil {
|
||||
t.Fatal("Expected wait function to be not nil")
|
||||
}
|
||||
|
||||
// There should an active fetcher for the chunk after the FetchFunc call
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil {
|
||||
t.Fatalf("Expected netStore to have one fetcher for the requested chunk")
|
||||
}
|
||||
|
||||
// wait function should timeout because we don't deliver the chunk with a Put
|
||||
err := wait(ctx)
|
||||
if err != context.DeadlineExceeded {
|
||||
t.Fatalf("Expected context.DeadLineExceeded err got %v", err)
|
||||
}
|
||||
|
||||
// the fetcher should be removed after timeout
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after timeout")
|
||||
}
|
||||
|
||||
// the fetcher context should be cancelled after timeout
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreFetchFuncAfterPut tests that the FetchFunc should return nil for a locally available chunk
|
||||
func TestNetStoreFetchFuncAfterPut(t *testing.T) {
|
||||
netStore, _, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// We deliver the created the chunk with a Put
|
||||
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no err got %v", err)
|
||||
}
|
||||
|
||||
// FetchFunc should return nil, because the chunk is available locally, no need to fetch it
|
||||
wait := netStore.FetchFunc(ctx, ch.Address())
|
||||
if wait != nil {
|
||||
t.Fatal("Expected wait to be nil")
|
||||
}
|
||||
|
||||
// No fetchers should be created at all
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to not have fetcher")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreGetCallsRequest tests if Get created a request on the NetFetcher for an unavailable chunk
|
||||
func TestNetStoreGetCallsRequest(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx := context.WithValue(context.Background(), "hopcount", uint8(5))
|
||||
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
// We call get for a not available chunk, it will timeout because the chunk is not delivered
|
||||
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
|
||||
|
||||
if err != context.DeadlineExceeded {
|
||||
t.Fatalf("Expected context.DeadlineExceeded err got %v", err)
|
||||
}
|
||||
|
||||
// NetStore should call NetFetcher.Request and wait for the chunk
|
||||
if !fetcher.requestCalled {
|
||||
t.Fatal("Expected NetFetcher.Request to be called")
|
||||
}
|
||||
|
||||
if fetcher.hopCounts[0] != 5 {
|
||||
t.Fatalf("Expected NetFetcher.Request be called with hopCount 5, got %v", fetcher.hopCounts[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreGetCallsOffer tests if Get created a request on the NetFetcher for an unavailable chunk
|
||||
// in case of a source peer provided in the context.
|
||||
func TestNetStoreGetCallsOffer(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
// If a source peer is added to the context, NetStore will handle it as an offer
|
||||
ctx := context.WithValue(context.Background(), "source", sourcePeerID.String())
|
||||
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
// We call get for a not available chunk, it will timeout because the chunk is not delivered
|
||||
_, err := netStore.Get(ctx, chunk.ModeGetRequest, ch.Address())
|
||||
|
||||
if err != context.DeadlineExceeded {
|
||||
t.Fatalf("Expect error %v got %v", context.DeadlineExceeded, err)
|
||||
}
|
||||
|
||||
// NetStore should call NetFetcher.Offer with the source peer
|
||||
if !fetcher.offerCalled {
|
||||
t.Fatal("Expected NetFetcher.Request to be called")
|
||||
}
|
||||
|
||||
if len(fetcher.sources) != 1 {
|
||||
t.Fatalf("Expected fetcher sources length 1 got %v", len(fetcher.sources))
|
||||
}
|
||||
|
||||
if fetcher.sources[0].String() != sourcePeerID.String() {
|
||||
t.Fatalf("Expected fetcher source %v got %v", sourcePeerID, fetcher.sources[0])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestNetStoreFetcherCountPeers tests multiple NetStore.Get calls with peer in the context.
|
||||
// There is no Put call, so the Get calls timeout
|
||||
func TestNetStoreFetcherCountPeers(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
addr := randomAddr()
|
||||
peers := []string{randomAddr().Hex(), randomAddr().Hex(), randomAddr().Hex()}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
errC := make(chan error)
|
||||
nrGets := 3
|
||||
|
||||
// Call Get 3 times with a peer in context
|
||||
for i := 0; i < nrGets; i++ {
|
||||
peer := peers[i]
|
||||
go func() {
|
||||
ctx := context.WithValue(ctx, "peer", peer)
|
||||
_, err := netStore.Get(ctx, chunk.ModeGetRequest, addr)
|
||||
errC <- err
|
||||
}()
|
||||
}
|
||||
|
||||
// All 3 Get calls should timeout
|
||||
for i := 0; i < nrGets; i++ {
|
||||
err := <-errC
|
||||
if err != context.DeadlineExceeded {
|
||||
t.Fatalf("Expected \"%v\" error got \"%v\"", context.DeadlineExceeded, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fetcher should be closed after timeout
|
||||
select {
|
||||
case <-fetcher.quit:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatalf("mockNetFetcher not closed after timeout")
|
||||
}
|
||||
|
||||
// All 3 peers should be given to NetFetcher after the 3 Get calls
|
||||
if len(fetcher.peersPerRequest) != nrGets {
|
||||
t.Fatalf("Expected 3 got %v", len(fetcher.peersPerRequest))
|
||||
}
|
||||
|
||||
for i, peers := range fetcher.peersPerRequest {
|
||||
if len(peers) < i+1 {
|
||||
t.Fatalf("Expected at least %v got %v", i+1, len(peers))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreFetchFuncCalledMultipleTimes calls the wait function given by FetchFunc three times,
|
||||
// and checks there is still exactly one fetcher for one chunk. Afthe chunk is delivered, it checks
|
||||
// if the fetcher is closed.
|
||||
func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ch := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
// FetchFunc should return a non-nil wait function, because the chunk is not available
|
||||
wait := netStore.FetchFunc(ctx, ch.Address())
|
||||
if wait == nil {
|
||||
t.Fatal("Expected wait function to be not nil")
|
||||
}
|
||||
|
||||
// There should be exactly one fetcher for the chunk
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
|
||||
t.Fatalf("Expected netStore to have one fetcher for the requested chunk")
|
||||
}
|
||||
|
||||
// Call wait three times in parallel
|
||||
count := 3
|
||||
errC := make(chan error)
|
||||
for i := 0; i < count; i++ {
|
||||
go func() {
|
||||
errC <- wait(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// sleep a little so the wait functions are called above
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// there should be still only one fetcher, because all wait calls are for the same chunk
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(ch.Address()) == nil {
|
||||
t.Fatal("Expected netStore to have one fetcher for the requested chunk")
|
||||
}
|
||||
|
||||
// Deliver the chunk with a Put
|
||||
_, err := netStore.Put(ctx, chunk.ModePutRequest, ch)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no err got %v", err)
|
||||
}
|
||||
|
||||
// wait until all wait calls return (because the chunk is delivered)
|
||||
for i := 0; i < count; i++ {
|
||||
err := <-errC
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// There should be no more fetchers for the delivered chunk
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after delivery")
|
||||
}
|
||||
|
||||
// The context for the fetcher should be cancelled after delivery
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetStoreFetcherLifeCycleWithTimeout is similar to TestNetStoreFetchFuncCalledMultipleTimes,
|
||||
// the only difference is that we don't deilver the chunk, just wait for timeout
|
||||
func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) {
|
||||
netStore, fetcher, cleanup := newTestNetStore(t)
|
||||
defer cleanup()
|
||||
|
||||
chunk := GenerateRandomChunk(chunk.DefaultSize)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// FetchFunc should return a non-nil wait function, because the chunk is not available
|
||||
wait := netStore.FetchFunc(ctx, chunk.Address())
|
||||
if wait == nil {
|
||||
t.Fatal("Expected wait function to be not nil")
|
||||
}
|
||||
|
||||
// There should be exactly one fetcher for the chunk
|
||||
if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil {
|
||||
t.Fatalf("Expected netStore to have one fetcher for the requested chunk")
|
||||
}
|
||||
|
||||
// Call wait three times in parallel
|
||||
count := 3
|
||||
errC := make(chan error)
|
||||
for i := 0; i < count; i++ {
|
||||
go func() {
|
||||
rctx, rcancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer rcancel()
|
||||
err := wait(rctx)
|
||||
if err != context.DeadlineExceeded {
|
||||
errC <- fmt.Errorf("Expected err %v got %v", context.DeadlineExceeded, err)
|
||||
return
|
||||
}
|
||||
errC <- nil
|
||||
}()
|
||||
}
|
||||
|
||||
// wait until all wait calls timeout
|
||||
for i := 0; i < count; i++ {
|
||||
err := <-errC
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// There should be no more fetchers after timeout
|
||||
if netStore.fetchers.Len() != 0 {
|
||||
t.Fatal("Expected netStore to remove the fetcher after delivery")
|
||||
}
|
||||
|
||||
// The context for the fetcher should be cancelled after timeout
|
||||
select {
|
||||
case <-fetcher.ctx.Done():
|
||||
default:
|
||||
t.Fatal("Expected fetcher context to be cancelled")
|
||||
}
|
||||
}
|
||||
|
||||
func randomAddr() Address {
|
||||
addr := make([]byte, 32)
|
||||
rand.Read(addr)
|
||||
return Address(addr)
|
||||
}
|
58
storage/request.go
Normal file
58
storage/request.go
Normal file
@ -0,0 +1,58 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ethersphere/swarm/network/timeouts"
|
||||
|
||||
"github.com/ethereum/go-ethereum/p2p/enode"
|
||||
)
|
||||
|
||||
// Request encapsulates all the necessary arguments when making a request to NetStore.
|
||||
// These could have also been added as part of the interface of NetStore.Get, but a request struct seemed
|
||||
// like a better option
|
||||
type Request struct {
|
||||
Addr Address // chunk address
|
||||
Origin enode.ID // who is sending us that request? we compare Origin to the suggested peer from RequestFromPeers
|
||||
PeersToSkip sync.Map // peers not to request chunk from
|
||||
}
|
||||
|
||||
// NewRequest returns a new instance of Request based on chunk address skip check and
|
||||
// a map of peers to skip.
|
||||
func NewRequest(addr Address) *Request {
|
||||
return &Request{
|
||||
Addr: addr,
|
||||
}
|
||||
}
|
||||
|
||||
// SkipPeer returns if the peer with nodeID should not be requested to deliver a chunk.
|
||||
// Peers to skip are kept per Request and for a time period of FailedPeerSkipDelay.
|
||||
func (r *Request) SkipPeer(nodeID string) bool {
|
||||
val, ok := r.PeersToSkip.Load(nodeID)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
t, ok := val.(time.Time)
|
||||
if ok && time.Now().After(t.Add(timeouts.FailedPeerSkipDelay)) {
|
||||
r.PeersToSkip.Delete(nodeID)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
Reference in New Issue
Block a user