gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #17796 from epiclabs-io/mru-feeds

Browse Source 
swarm/storage/feeds: Renamed MRU to Swarm Feeds
This commit is contained in:
Viktor Trón
2018-10-03 14:59:41 +02:00
committed by GitHub
parent 303b99663e de01178c18
commit e5677114dc
44 changed files with 740 additions and 812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
swarm/storage/mru/binaryserializer.go → swarm/storage/feed/binaryserializer.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import "github.com/ethereum/go-ethereum/common/hexutil"

2
swarm/storage/mru/binaryserializer_test.go → swarm/storage/feed/binaryserializer_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"encoding/json"

14
swarm/storage/mru/cacheentry.go → swarm/storage/feed/cacheentry.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"bytes"
@ -26,23 +26,23 @@ import (
const (
hasherCount = 8
resourceHashAlgorithm = storage.SHA3Hash
feedsHashAlgorithm = storage.SHA3Hash
defaultRetrieveTimeout = 100 * time.Millisecond
)
// cacheEntry caches resource data and the metadata of its root chunk.
// cacheEntry caches the last known update of a specific Swarm feed.
type cacheEntry struct {
ResourceUpdate
Update
*bytes.Reader
lastKey storage.Address
}
// implements storage.LazySectionReader
func (r *cacheEntry) Size(ctx context.Context, _ chan bool) (int64, error) {
return int64(len(r.ResourceUpdate.data)), nil
return int64(len(r.Update.data)), nil
}
//returns the resource's topic
//returns the feed's topic
func (r *cacheEntry) Topic() Topic {
return r.View.Topic
return r.Feed.Topic
}

43
swarm/storage/feed/doc.go Normal file
View File

@ -0,0 +1,43 @@
/*
Package feeds defines Swarm Feeds.
Swarm Feeds allows a user to build an update feed about a particular topic
without resorting to ENS on each update.
The update scheme is built on swarm chunks with chunk keys following
a predictable, versionable pattern.
A Feed is tied to a unique identifier that is deterministically generated out of
the chosen topic.
A Feed is defined as the series of updates of a specific user about a particular topic
Actual data updates are also made in the form of swarm chunks. The keys
of the updates are the hash of a concatenation of properties as follows:
updateAddr = H(Feed, Epoch ID)
where H is the SHA3 hash function
Feed is the combination of Topic and the user address
Epoch ID is a time slot. See the lookup package for more information.
A user looking up a the latest update in a Feed only needs to know the Topic
and the other user's address.
The Feed Update data is:
updatedata = Feed|Epoch|data
The full update data that goes in the chunk payload is:
updatedata|sign(updatedata)
Structure Summary:
Request: Feed Update with signature
Update: headers + data
Header: Protocol version and reserved for future use placeholders
ID: Information about how to locate a specific update
Feed: Represents a user's series of publications about a specific Topic
Topic: Item that the updates are about
User: User who updates the Feed
Epoch: time slot where the update is stored
*/
package feed

8
swarm/storage/mru/error.go → swarm/storage/feed/error.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"fmt"
@ -35,7 +35,7 @@ const (
ErrCnt
)
// Error is a the typed error object used for Mutable Resources
// Error is a the typed error object used for Swarm feeds
type Error struct {
code int
err string
@ -47,12 +47,12 @@ func (e *Error) Error() string {
}
// Code returns the error code
// Error codes are enumerated in the error.go file within the mru package
// Error codes are enumerated in the error.go file within the feeds package
func (e *Error) Code() int {
return e.code
}
// NewError creates a new Mutable Resource Error object with the specified code and custom error message
// NewError creates a new Swarm feeds Error object with the specified code and custom error message
func NewError(code int, s string) error {
if code < 0 || code >= ErrCnt {
panic("no such error code!")

66
swarm/storage/mru/view.go → swarm/storage/feed/feed.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"hash"
@ -25,21 +25,21 @@ import (
"github.com/ethereum/go-ethereum/swarm/storage"
)
// View represents a particular user's view of a resource
type View struct {
// Feed represents a particular user's stream of updates on a topic
type Feed struct {
Topic Topic `json:"topic"`
User common.Address `json:"user"`
}
// View layout:
// Feed layout:
// TopicLength bytes
// userAddr common.AddressLength bytes
const viewLength = TopicLength + common.AddressLength
const feedLength = TopicLength + common.AddressLength
// mapKey calculates a unique id for this view for the cache map in `Handler`
func (u *View) mapKey() uint64 {
serializedData := make([]byte, viewLength)
u.binaryPut(serializedData)
// mapKey calculates a unique id for this feed. Used by the cache map in `Handler`
func (f *Feed) mapKey() uint64 {
serializedData := make([]byte, feedLength)
f.binaryPut(serializedData)
hasher := hashPool.Get().(hash.Hash)
defer hashPool.Put(hasher)
hasher.Reset()
@ -48,55 +48,55 @@ func (u *View) mapKey() uint64 {
return *(*uint64)(unsafe.Pointer(&hash[0]))
}
// binaryPut serializes this View instance into the provided slice
func (u *View) binaryPut(serializedData []byte) error {
if len(serializedData) != viewLength {
return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize View. Expected %d, got %d", viewLength, len(serializedData))
// binaryPut serializes this feed instance into the provided slice
func (f *Feed) binaryPut(serializedData []byte) error {
if len(serializedData) != feedLength {
return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize feed. Expected %d, got %d", feedLength, len(serializedData))
}
var cursor int
copy(serializedData[cursor:cursor+TopicLength], u.Topic[:TopicLength])
copy(serializedData[cursor:cursor+TopicLength], f.Topic[:TopicLength])
cursor += TopicLength
copy(serializedData[cursor:cursor+common.AddressLength], u.User[:])
copy(serializedData[cursor:cursor+common.AddressLength], f.User[:])
cursor += common.AddressLength
return nil
}
// binaryLength returns the expected size of this structure when serialized
func (u *View) binaryLength() int {
return viewLength
func (f *Feed) binaryLength() int {
return feedLength
}
// binaryGet restores the current instance from the information contained in the passed slice
func (u *View) binaryGet(serializedData []byte) error {
if len(serializedData) != viewLength {
return NewErrorf(ErrInvalidValue, "Incorrect slice size to read View. Expected %d, got %d", viewLength, len(serializedData))
func (f *Feed) binaryGet(serializedData []byte) error {
if len(serializedData) != feedLength {
return NewErrorf(ErrInvalidValue, "Incorrect slice size to read feed. Expected %d, got %d", feedLength, len(serializedData))
}
var cursor int
copy(u.Topic[:], serializedData[cursor:cursor+TopicLength])
copy(f.Topic[:], serializedData[cursor:cursor+TopicLength])
cursor += TopicLength
copy(u.User[:], serializedData[cursor:cursor+common.AddressLength])
copy(f.User[:], serializedData[cursor:cursor+common.AddressLength])
cursor += common.AddressLength
return nil
}
// Hex serializes the View to a hex string
func (u *View) Hex() string {
serializedData := make([]byte, viewLength)
u.binaryPut(serializedData)
// Hex serializes the feed to a hex string
func (f *Feed) Hex() string {
serializedData := make([]byte, feedLength)
f.binaryPut(serializedData)
return hexutil.Encode(serializedData)
}
// FromValues deserializes this instance from a string key-value store
// useful to parse query strings
func (u *View) FromValues(values Values) (err error) {
func (f *Feed) FromValues(values Values) (err error) {
topic := values.Get("topic")
if topic != "" {
if err := u.Topic.FromHex(values.Get("topic")); err != nil {
if err := f.Topic.FromHex(values.Get("topic")); err != nil {
return err
}
} else { // see if the user set name and relatedcontent
@ -108,18 +108,18 @@ func (u *View) FromValues(values Values) (err error) {
}
relatedContent = relatedContent[:storage.AddressLength]
}
u.Topic, err = NewTopic(name, relatedContent)
f.Topic, err = NewTopic(name, relatedContent)
if err != nil {
return err
}
}
u.User = common.HexToAddress(values.Get("user"))
f.User = common.HexToAddress(values.Get("user"))
return nil
}
// AppendValues serializes this structure into the provided string key-value store
// useful to build query strings
func (u *View) AppendValues(values Values) {
values.Set("topic", u.Topic.Hex())
values.Set("user", u.User.Hex())
func (f *Feed) AppendValues(values Values) {
values.Set("topic", f.Topic.Hex())
values.Set("user", f.User.Hex())
}

14
swarm/storage/mru/view_test.go → swarm/storage/feed/feed_test.go
View File

@ -13,24 +13,24 @@
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"testing"
)
func getTestView() *View {
func getTestFeed() *Feed {
topic, _ := NewTopic("world news report, every hour", nil)
return &View{
return &Feed{
Topic: topic,
User: newCharlieSigner().Address(),
}
}
func TestViewSerializerDeserializer(t *testing.T) {
testBinarySerializerRecovery(t, getTestView(), "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781c")
func TestFeedSerializerDeserializer(t *testing.T) {
testBinarySerializerRecovery(t, getTestFeed(), "0x776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781c")
}
func TestMetadataSerializerLengthCheck(t *testing.T) {
testBinarySerializerLengthCheck(t, getTestView())
func TestFeedSerializerLengthCheck(t *testing.T) {
testBinarySerializerLengthCheck(t, getTestFeed())
}

175
swarm/storage/mru/handler.go → swarm/storage/feed/handler.go
View File

@ -14,9 +14,9 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Handler is the API for Mutable Resources
// It enables creating, updating, syncing and retrieving resources and their update data
package mru
// Handler is the API for feeds
// It enables creating, updating, syncing and retrieving feed updates and their data
package feed
import (
"bytes"
@ -25,7 +25,7 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage"
@ -34,8 +34,8 @@ import (
type Handler struct {
chunkStore *storage.NetStore
HashSize int
resources map[uint64]*cacheEntry
resourceLock sync.RWMutex
cache map[uint64]*cacheEntry
cacheLock sync.RWMutex
storeTimeout time.Duration
queryMaxPeriods uint32
}
@ -52,35 +52,35 @@ var hashPool sync.Pool
func init() {
hashPool = sync.Pool{
New: func() interface{} {
return storage.MakeHashFunc(resourceHashAlgorithm)()
return storage.MakeHashFunc(feedsHashAlgorithm)()
},
}
}
// NewHandler creates a new Mutable Resource API
// NewHandler creates a new Swarm feeds API
func NewHandler(params *HandlerParams) *Handler {
rh := &Handler{
resources: make(map[uint64]*cacheEntry),
fh := &Handler{
cache: make(map[uint64]*cacheEntry),
}
for i := 0; i < hasherCount; i++ {
hashfunc := storage.MakeHashFunc(resourceHashAlgorithm)()
if rh.HashSize == 0 {
rh.HashSize = hashfunc.Size()
hashfunc := storage.MakeHashFunc(feedsHashAlgorithm)()
if fh.HashSize == 0 {
fh.HashSize = hashfunc.Size()
}
hashPool.Put(hashfunc)
}
return rh
return fh
}
// SetStore sets the store backend for the Mutable Resource API
// SetStore sets the store backend for the Swarm feeds API
func (h *Handler) SetStore(store *storage.NetStore) {
h.chunkStore = store
}
// Validate is a chunk validation method
// If it looks like a resource update, the chunk address is checked against the userAddr of the update's signature
// If it looks like a feed update, the chunk address is checked against the userAddr of the update's signature
// It implements the storage.ChunkValidator interface
func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
dataLength := len(data)
@ -89,67 +89,67 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool {
}
// check if it is a properly formatted update chunk with
// valid signature and proof of ownership of the resource it is trying
// valid signature and proof of ownership of the feed it is trying
// to update
// First, deserialize the chunk
var r Request
if err := r.fromChunk(chunkAddr, data); err != nil {
log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error())
log.Debug("Invalid feed update chunk", "addr", chunkAddr.Hex(), "err", err.Error())
return false
}
// Verify signatures and that the signer actually owns the resource
// Verify signatures and that the signer actually owns the feed
// If it fails, it means either the signature is not valid, data is corrupted
// or someone is trying to update someone else's resource.
// or someone is trying to update someone else's feed.
if err := r.Verify(); err != nil {
log.Debug("Invalid signature", "err", err)
log.Debug("Invalid feed update signature", "err", err)
return false
}
return true
}
// GetContent retrieves the data payload of the last synced update of the Mutable Resource
func (h *Handler) GetContent(view *View) (storage.Address, []byte, error) {
if view == nil {
return nil, nil, NewError(ErrInvalidValue, "view is nil")
// GetContent retrieves the data payload of the last synced update of the feed
func (h *Handler) GetContent(feed *Feed) (storage.Address, []byte, error) {
if feed == nil {
return nil, nil, NewError(ErrInvalidValue, "feed is nil")
}
rsrc := h.get(view)
if rsrc == nil {
return nil, nil, NewError(ErrNotFound, "resource does not exist")
feedUpdate := h.get(feed)
if feedUpdate == nil {
return nil, nil, NewError(ErrNotFound, "feed update not cached")
}
return rsrc.lastKey, rsrc.data, nil
return feedUpdate.lastKey, feedUpdate.data, nil
}
// NewRequest prepares a Request structure with all the necessary information to
// just add the desired data and sign it.
// The resulting structure can then be signed and passed to Handler.Update to be verified and sent
func (h *Handler) NewRequest(ctx context.Context, view *View) (request *Request, err error) {
if view == nil {
return nil, NewError(ErrInvalidValue, "view cannot be nil")
func (h *Handler) NewRequest(ctx context.Context, feed *Feed) (request *Request, err error) {
if feed == nil {
return nil, NewError(ErrInvalidValue, "feed cannot be nil")
}
now := TimestampProvider.Now().Time
request = new(Request)
request.Header.Version = ProtocolVersion
query := NewQueryLatest(view, lookup.NoClue)
query := NewQueryLatest(feed, lookup.NoClue)
rsrc, err := h.Lookup(ctx, query)
feedUpdate, err := h.Lookup(ctx, query)
if err != nil {
if err.(*Error).code != ErrNotFound {
return nil, err
}
// not finding updates means that there is a network error
// or that the resource really does not have updates
// or that the feed really does not have updates
}
request.View = *view
request.Feed = *feed
// if we already have an update, then find next epoch
if rsrc != nil {
request.Epoch = lookup.GetNextEpoch(rsrc.Epoch, now)
if feedUpdate != nil {
request.Epoch = lookup.GetNextEpoch(feedUpdate.Epoch, now)
} else {
request.Epoch = lookup.GetFirstEpoch(now)
}
@ -157,13 +157,10 @@ func (h *Handler) NewRequest(ctx context.Context, view *View) (request *Request,
return request, nil
}
// Lookup retrieves a specific or latest version of the resource
// Lookup works differently depending on the configuration of `ID`
// See the `ID` documentation and helper functions:
// `LookupLatest` and `LookupBefore`
// When looking for the latest update, it starts at the next period after the current time.
// upon failure tries the corresponding keys of each previous period until one is found
// (or startTime is reached, in which case there are no updates).
// Lookup retrieves a specific or latest feed update
// Lookup works differently depending on the configuration of `query`
// See the `query` documentation and helper functions:
// `NewQueryLatest` and `NewQuery`
func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) {
timeLimit := query.TimeLimit
@ -172,7 +169,7 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error)
}
if query.Hint == lookup.NoClue { // try to use our cache
entry := h.get(&query.View)
entry := h.get(&query.Feed)
if entry != nil && entry.Epoch.Time <= timeLimit { // avoid bad hints
query.Hint = entry.Epoch
}
@ -183,19 +180,19 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error)
return nil, NewError(ErrInit, "Call Handler.SetStore() before performing lookups")
}
var ul ID
ul.View = query.View
var id ID
id.Feed = query.Feed
var readCount int
// Invoke the lookup engine.
// The callback will be called every time the lookup algorithm needs to guess
requestPtr, err := lookup.Lookup(timeLimit, query.Hint, func(epoch lookup.Epoch, now uint64) (interface{}, error) {
readCount++
ul.Epoch = epoch
id.Epoch = epoch
ctx, cancel := context.WithTimeout(ctx, defaultRetrieveTimeout)
defer cancel()
chunk, err := h.chunkStore.Get(ctx, ul.Addr())
chunk, err := h.chunkStore.Get(ctx, id.Addr())
if err != nil { // TODO: check for catastrophic errors other than chunk not found
return nil, nil
}
@ -213,39 +210,39 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error)
return nil, err
}
log.Info(fmt.Sprintf("Resource lookup finished in %d lookups", readCount))
log.Info(fmt.Sprintf("Feed lookup finished in %d lookups", readCount))
request, _ := requestPtr.(*Request)
if request == nil {
return nil, NewError(ErrNotFound, "no updates found")
return nil, NewError(ErrNotFound, "no feed updates found")
}
return h.updateCache(request)
}
// update mutable resource cache map with specified content
// update feed updates cache with specified content
func (h *Handler) updateCache(request *Request) (*cacheEntry, error) {
updateAddr := request.Addr()
log.Trace("resource cache update", "topic", request.Topic.Hex(), "updatekey", updateAddr, "epoch time", request.Epoch.Time, "epoch level", request.Epoch.Level)
log.Trace("feed cache update", "topic", request.Topic.Hex(), "updateaddr", updateAddr, "epoch time", request.Epoch.Time, "epoch level", request.Epoch.Level)
rsrc := h.get(&request.View)
if rsrc == nil {
rsrc = &cacheEntry{}
h.set(&request.View, rsrc)
feedUpdate := h.get(&request.Feed)
if feedUpdate == nil {
feedUpdate = &cacheEntry{}
h.set(&request.Feed, feedUpdate)
}
// update our rsrcs entry map
rsrc.lastKey = updateAddr
rsrc.ResourceUpdate = request.ResourceUpdate
rsrc.Reader = bytes.NewReader(rsrc.data)
return rsrc, nil
feedUpdate.lastKey = updateAddr
feedUpdate.Update = request.Update
feedUpdate.Reader = bytes.NewReader(feedUpdate.data)
return feedUpdate, nil
}
// Update adds an actual data update
// Uses the Mutable Resource metadata currently loaded in the resources map entry.
// It is the caller's responsibility to make sure that this data is not stale.
// Note that a Mutable Resource update cannot span chunks, and thus has a MAX NET LENGTH 4096, INCLUDING update header data and signature. An error will be returned if the total length of the chunk payload will exceed this limit.
// Update publishes a feed update
// Note that a feed update cannot span chunks, and thus has a MAX NET LENGTH 4096, INCLUDING update header data and signature.
// This results in a max payload of `maxUpdateDataLength` (check update.go for more details)
// An error will be returned if the total length of the chunk payload will exceed this limit.
// Update can only check if the caller is trying to overwrite the very last known version, otherwise it just puts the update
// on the network.
func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Address, err error) {
@ -255,8 +252,8 @@ func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Ad
return nil, NewError(ErrInit, "Call Handler.SetStore() before updating")
}
rsrc := h.get(&r.View)
if rsrc != nil && rsrc.Epoch.Equals(r.Epoch) { // This is the only cheap check we can do for sure
feedUpdate := h.get(&r.Feed)
if feedUpdate != nil && feedUpdate.Epoch.Equals(r.Epoch) { // This is the only cheap check we can do for sure
return nil, NewError(ErrInvalidValue, "A former update in this epoch is already known to exist")
}
@ -267,32 +264,32 @@ func (h *Handler) Update(ctx context.Context, r *Request) (updateAddr storage.Ad
// send the chunk
h.chunkStore.Put(ctx, chunk)
log.Trace("resource update", "updateAddr", r.idAddr, "epoch time", r.Epoch.Time, "epoch level", r.Epoch.Level, "data", chunk.Data())
// update our resources map cache entry if the new update is older than the one we have, if we have it.
if rsrc != nil && r.Epoch.After(rsrc.Epoch) {
rsrc.Epoch = r.Epoch
rsrc.data = make([]byte, len(r.data))
rsrc.lastKey = r.idAddr
copy(rsrc.data, r.data)
rsrc.Reader = bytes.NewReader(rsrc.data)
log.Trace("feed update", "updateAddr", r.idAddr, "epoch time", r.Epoch.Time, "epoch level", r.Epoch.Level, "data", chunk.Data())
// update our feed updates map cache entry if the new update is older than the one we have, if we have it.
if feedUpdate != nil && r.Epoch.After(feedUpdate.Epoch) {
feedUpdate.Epoch = r.Epoch
feedUpdate.data = make([]byte, len(r.data))
feedUpdate.lastKey = r.idAddr
copy(feedUpdate.data, r.data)
feedUpdate.Reader = bytes.NewReader(feedUpdate.data)
}
return r.idAddr, nil
}
// Retrieves the resource cache value for the given nameHash
func (h *Handler) get(view *View) *cacheEntry {
mapKey := view.mapKey()
h.resourceLock.RLock()
defer h.resourceLock.RUnlock()
rsrc := h.resources[mapKey]
return rsrc
// Retrieves the feed update cache value for the given nameHash
func (h *Handler) get(feed *Feed) *cacheEntry {
mapKey := feed.mapKey()
h.cacheLock.RLock()
defer h.cacheLock.RUnlock()
feedUpdate := h.cache[mapKey]
return feedUpdate
}
// Sets the resource cache value for the given View
func (h *Handler) set(view *View, rsrc *cacheEntry) {
mapKey := view.mapKey()
h.resourceLock.Lock()
defer h.resourceLock.Unlock()
h.resources[mapKey] = rsrc
// Sets the feed update cache value for the given feed
func (h *Handler) set(feed *Feed, feedUpdate *cacheEntry) {
mapKey := feed.mapKey()
h.cacheLock.Lock()
defer h.cacheLock.Unlock()
h.cache[mapKey] = feedUpdate
}

130
swarm/storage/mru/handler_test.go → swarm/storage/feed/handler_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"bytes"
@ -31,7 +31,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
var (
@ -40,7 +40,7 @@ var (
Time: uint64(4200),
}
cleanF func()
resourceName = "føø.bar"
subtopicName = "føø.bar"
hashfunc = storage.MakeHashFunc(storage.DefaultHash)
)
@ -73,7 +73,7 @@ func (f *fakeTimeProvider) Now() Timestamp {
}
// make updates and retrieve them based on periods and versions
func TestResourceHandler(t *testing.T) {
func TestFeedsHandler(t *testing.T) {
// make fake timeProvider
clock := &fakeTimeProvider{
@ -83,18 +83,18 @@ func TestResourceHandler(t *testing.T) {
// signer containing private key
signer := newAliceSigner()
rh, datadir, teardownTest, err := setupTest(clock, signer)
feedsHandler, datadir, teardownTest, err := setupTest(clock, signer)
if err != nil {
t.Fatal(err)
}
defer teardownTest()
// create a new resource
// create a new feed
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
topic, _ := NewTopic("Mess with mru code and see what ghost catches you", nil)
view := View{
topic, _ := NewTopic("Mess with Swarm feeds code and see what ghost catches you", nil)
fd := Feed{
Topic: topic,
User: signer.Address(),
}
@ -107,14 +107,14 @@ func TestResourceHandler(t *testing.T) {
"clyde", // t=4285
}
request := NewFirstRequest(view.Topic) // this timestamps the update at t = 4200 (start time)
resourcekey := make(map[string]storage.Address)
request := NewFirstRequest(fd.Topic) // this timestamps the update at t = 4200 (start time)
chunkAddress := make(map[string]storage.Address)
data := []byte(updates[0])
request.SetData(data)
if err := request.Sign(signer); err != nil {
t.Fatal(err)
}
resourcekey[updates[0]], err = rh.Update(ctx, request)
chunkAddress[updates[0]], err = feedsHandler.Update(ctx, request)
if err != nil {
t.Fatal(err)
}
@ -122,7 +122,7 @@ func TestResourceHandler(t *testing.T) {
// move the clock ahead 21 seconds
clock.FastForward(21) // t=4221
request, err = rh.NewRequest(ctx, &request.View) // this timestamps the update at t = 4221
request, err = feedsHandler.NewRequest(ctx, &request.Feed) // this timestamps the update at t = 4221
if err != nil {
t.Fatal(err)
}
@ -136,14 +136,14 @@ func TestResourceHandler(t *testing.T) {
if err := request.Sign(signer); err != nil {
t.Fatal(err)
}
resourcekey[updates[1]], err = rh.Update(ctx, request)
chunkAddress[updates[1]], err = feedsHandler.Update(ctx, request)
if err == nil {
t.Fatal("Expected update to fail since an update in this epoch already exists")
}
// move the clock ahead 21 seconds
clock.FastForward(21) // t=4242
request, err = rh.NewRequest(ctx, &request.View)
request, err = feedsHandler.NewRequest(ctx, &request.Feed)
if err != nil {
t.Fatal(err)
}
@ -151,14 +151,14 @@ func TestResourceHandler(t *testing.T) {
if err := request.Sign(signer); err != nil {
t.Fatal(err)
}
resourcekey[updates[1]], err = rh.Update(ctx, request)
chunkAddress[updates[1]], err = feedsHandler.Update(ctx, request)
if err != nil {
t.Fatal(err)
}
// move the clock ahead 42 seconds
clock.FastForward(42) // t=4284
request, err = rh.NewRequest(ctx, &request.View)
request, err = feedsHandler.NewRequest(ctx, &request.Feed)
if err != nil {
t.Fatal(err)
}
@ -167,14 +167,14 @@ func TestResourceHandler(t *testing.T) {
if err := request.Sign(signer); err != nil {
t.Fatal(err)
}
resourcekey[updates[2]], err = rh.Update(ctx, request)
chunkAddress[updates[2]], err = feedsHandler.Update(ctx, request)
if err != nil {
t.Fatal(err)
}
// move the clock ahead 1 second
clock.FastForward(1) // t=4285
request, err = rh.NewRequest(ctx, &request.View)
request, err = feedsHandler.NewRequest(ctx, &request.Feed)
if err != nil {
t.Fatal(err)
}
@ -187,56 +187,56 @@ func TestResourceHandler(t *testing.T) {
if err := request.Sign(signer); err != nil {
t.Fatal(err)
}
resourcekey[updates[3]], err = rh.Update(ctx, request)
chunkAddress[updates[3]], err = feedsHandler.Update(ctx, request)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
rh.Close()
feedsHandler.Close()
// check we can retrieve the updates after close
clock.FastForward(2000) // t=6285
rhparams := &HandlerParams{}
feedParams := &HandlerParams{}
rh2, err := NewTestHandler(datadir, rhparams)
feedsHandler2, err := NewTestHandler(datadir, feedParams)
if err != nil {
t.Fatal(err)
}
rsrc2, err := rh2.Lookup(ctx, NewQueryLatest(&request.View, lookup.NoClue))
update2, err := feedsHandler2.Lookup(ctx, NewQueryLatest(&request.Feed, lookup.NoClue))
if err != nil {
t.Fatal(err)
}
// last update should be "clyde"
if !bytes.Equal(rsrc2.data, []byte(updates[len(updates)-1])) {
t.Fatalf("resource data was %v, expected %v", string(rsrc2.data), updates[len(updates)-1])
if !bytes.Equal(update2.data, []byte(updates[len(updates)-1])) {
t.Fatalf("feed update data was %v, expected %v", string(update2.data), updates[len(updates)-1])
}
if rsrc2.Level != 22 {
t.Fatalf("resource epoch level was %d, expected 22", rsrc2.Level)
if update2.Level != 22 {
t.Fatalf("feed update epoch level was %d, expected 22", update2.Level)
}
if rsrc2.Base() != 0 {
t.Fatalf("resource epoch base time was %d, expected 0", rsrc2.Base())
if update2.Base() != 0 {
t.Fatalf("feed update epoch base time was %d, expected 0", update2.Base())
}
log.Debug("Latest lookup", "epoch base time", rsrc2.Base(), "epoch level", rsrc2.Level, "data", rsrc2.data)
log.Debug("Latest lookup", "epoch base time", update2.Base(), "epoch level", update2.Level, "data", update2.data)
// specific point in time
rsrc, err := rh2.Lookup(ctx, NewQuery(&request.View, 4284, lookup.NoClue))
update, err := feedsHandler2.Lookup(ctx, NewQuery(&request.Feed, 4284, lookup.NoClue))
if err != nil {
t.Fatal(err)
}
// check data
if !bytes.Equal(rsrc.data, []byte(updates[2])) {
t.Fatalf("resource data (historical) was %v, expected %v", string(rsrc2.data), updates[2])
if !bytes.Equal(update.data, []byte(updates[2])) {
t.Fatalf("feed update data (historical) was %v, expected %v", string(update2.data), updates[2])
}
log.Debug("Historical lookup", "epoch base time", rsrc2.Base(), "epoch level", rsrc2.Level, "data", rsrc2.data)
log.Debug("Historical lookup", "epoch base time", update2.Base(), "epoch level", update2.Level, "data", update2.data)
// beyond the first should yield an error
rsrc, err = rh2.Lookup(ctx, NewQuery(&request.View, startTime.Time-1, lookup.NoClue))
update, err = feedsHandler2.Lookup(ctx, NewQuery(&request.Feed, startTime.Time-1, lookup.NoClue))
if err == nil {
t.Fatalf("expected previous to fail, returned epoch %s data %v", rsrc.Epoch.String(), rsrc.data)
t.Fatalf("expected previous to fail, returned epoch %s data %v", update.Epoch.String(), update.data)
}
}
@ -266,11 +266,11 @@ func TestSparseUpdates(t *testing.T) {
defer teardownTest()
defer os.RemoveAll(datadir)
// create a new resource
// create a new feed
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
topic, _ := NewTopic("Very slow updates", nil)
view := View{
fd := Feed{
Topic: topic,
User: signer.Address(),
}
@ -280,7 +280,7 @@ func TestSparseUpdates(t *testing.T) {
var epoch lookup.Epoch
var lastUpdateTime uint64
for T := uint64(0); T < today; T += 5 * Year {
request := NewFirstRequest(view.Topic)
request := NewFirstRequest(fd.Topic)
request.Epoch = lookup.GetNextEpoch(epoch, T)
request.data = generateData(T) // this generates some data that depends on T, so we can check later
request.Sign(signer)
@ -295,14 +295,14 @@ func TestSparseUpdates(t *testing.T) {
lastUpdateTime = T
}
query := NewQuery(&view, today, lookup.NoClue)
query := NewQuery(&fd, today, lookup.NoClue)
_, err = rh.Lookup(ctx, query)
if err != nil {
t.Fatal(err)
}
_, content, err := rh.GetContent(&view)
_, content, err := rh.GetContent(&fd)
if err != nil {
t.Fatal(err)
}
@ -321,7 +321,7 @@ func TestSparseUpdates(t *testing.T) {
t.Fatal(err)
}
_, content, err = rh.GetContent(&view)
_, content, err = rh.GetContent(&fd)
if err != nil {
t.Fatal(err)
}
@ -348,13 +348,13 @@ func TestValidator(t *testing.T) {
}
defer teardownTest()
// create new resource
topic, _ := NewTopic(resourceName, nil)
view := View{
// create new feed
topic, _ := NewTopic(subtopicName, nil)
fd := Feed{
Topic: topic,
User: signer.Address(),
}
mr := NewFirstRequest(view.Topic)
mr := NewFirstRequest(fd.Topic)
// chunk with address
data := []byte("foo")
@ -382,7 +382,7 @@ func TestValidator(t *testing.T) {
}
// tests that the content address validator correctly checks the data
// tests that resource update chunks are passed through content address validator
// tests that feed update chunks are passed through content address validator
// there is some redundancy in this test as it also tests content addressed chunks,
// which should be evaluated as invalid chunks by this validator
func TestValidatorInStore(t *testing.T) {
@ -396,7 +396,7 @@ func TestValidatorInStore(t *testing.T) {
signer := newAliceSigner()
// set up localstore
datadir, err := ioutil.TempDir("", "storage-testresourcevalidator")
datadir, err := ioutil.TempDir("", "storage-testfeedsvalidator")
if err != nil {
t.Fatal(err)
}
@ -409,10 +409,10 @@ func TestValidatorInStore(t *testing.T) {
t.Fatal(err)
}
// set up resource handler and add is as a validator to the localstore
rhParams := &HandlerParams{}
rh := NewHandler(rhParams)
store.Validators = append(store.Validators, rh)
// set up Swarm feeds handler and add is as a validator to the localstore
fhParams := &HandlerParams{}
fh := NewHandler(fhParams)
store.Validators = append(store.Validators, fh)
// create content addressed chunks, one good, one faulty
chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2)
@ -420,17 +420,17 @@ func TestValidatorInStore(t *testing.T) {
badChunk := storage.NewChunk(chunks[1].Address(), goodChunk.Data())
topic, _ := NewTopic("xyzzy", nil)
view := View{
fd := Feed{
Topic: topic,
User: signer.Address(),
}
// create a resource update chunk with correct publickey
// create a feed update chunk with correct publickey
id := ID{
Epoch: lookup.Epoch{Time: 42,
Level: 1,
},
View: view,
Feed: fd,
}
updateAddr := id.Addr()
@ -438,7 +438,7 @@ func TestValidatorInStore(t *testing.T) {
r := new(Request)
r.idAddr = updateAddr
r.ResourceUpdate.ID = id
r.Update.ID = id
r.data = data
r.Sign(signer)
@ -451,20 +451,20 @@ func TestValidatorInStore(t *testing.T) {
// put the chunks in the store and check their error status
err = store.Put(context.Background(), goodChunk)
if err == nil {
t.Fatal("expected error on good content address chunk with resource validator only, but got nil")
t.Fatal("expected error on good content address chunk with feed update validator only, but got nil")
}
err = store.Put(context.Background(), badChunk)
if err == nil {
t.Fatal("expected error on bad content address chunk with resource validator only, but got nil")
t.Fatal("expected error on bad content address chunk with feed update validator only, but got nil")
}
err = store.Put(context.Background(), uglyChunk)
if err != nil {
t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err)
t.Fatalf("expected no error on feed update chunk with feed update validator only, but got: %s", err)
}
}
// create rpc and resourcehandler
func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler, datadir string, teardown func(), err error) {
// create rpc and feeds Handler
func setupTest(timeProvider timestampProvider, signer Signer) (fh *TestHandler, datadir string, teardown func(), err error) {
var fsClean func()
var rpcClean func()
@ -478,7 +478,7 @@ func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler,
}
// temp datadir
datadir, err = ioutil.TempDir("", "rh")
datadir, err = ioutil.TempDir("", "fh")
if err != nil {
return nil, "", nil, err
}
@ -487,9 +487,9 @@ func setupTest(timeProvider timestampProvider, signer Signer) (rh *TestHandler,
}
TimestampProvider = timeProvider
rhparams := &HandlerParams{}
rh, err = NewTestHandler(datadir, rhparams)
return rh, datadir, cleanF, err
fhParams := &HandlerParams{}
fh, err = NewTestHandler(datadir, fhParams)
return fh, datadir, cleanF, err
}
func newAliceSigner() *GenericSigner {

30
swarm/storage/mru/id.go → swarm/storage/feed/id.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"fmt"
@ -22,28 +22,28 @@ import (
"strconv"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// ID uniquely identifies an update on the network.
type ID struct {
View `json:"view"`
Feed `json:"feed"`
lookup.Epoch `json:"epoch"`
}
// ID layout:
// View viewLength bytes
// Feed feedLength bytes
// Epoch EpochLength
const idLength = viewLength + lookup.EpochLength
const idLength = feedLength + lookup.EpochLength
// Addr calculates the resource update chunk address corresponding to this ID
// Addr calculates the feed update chunk address corresponding to this ID
func (u *ID) Addr() (updateAddr storage.Address) {
serializedData := make([]byte, idLength)
var cursor int
u.View.binaryPut(serializedData[cursor : cursor+viewLength])
cursor += viewLength
u.Feed.binaryPut(serializedData[cursor : cursor+feedLength])
cursor += feedLength
eid := u.Epoch.ID()
copy(serializedData[cursor:cursor+lookup.EpochLength], eid[:])
@ -61,10 +61,10 @@ func (u *ID) binaryPut(serializedData []byte) error {
return NewErrorf(ErrInvalidValue, "Incorrect slice size to serialize ID. Expected %d, got %d", idLength, len(serializedData))
}
var cursor int
if err := u.View.binaryPut(serializedData[cursor : cursor+viewLength]); err != nil {
if err := u.Feed.binaryPut(serializedData[cursor : cursor+feedLength]); err != nil {
return err
}
cursor += viewLength
cursor += feedLength
epochBytes, err := u.Epoch.MarshalBinary()
if err != nil {
@ -88,10 +88,10 @@ func (u *ID) binaryGet(serializedData []byte) error {
}
var cursor int
if err := u.View.binaryGet(serializedData[cursor : cursor+viewLength]); err != nil {
if err := u.Feed.binaryGet(serializedData[cursor : cursor+feedLength]); err != nil {
return err
}
cursor += viewLength
cursor += feedLength
if err := u.Epoch.UnmarshalBinary(serializedData[cursor : cursor+lookup.EpochLength]); err != nil {
return err
@ -108,8 +108,8 @@ func (u *ID) FromValues(values Values) error {
u.Epoch.Level = uint8(level)
u.Epoch.Time, _ = strconv.ParseUint(values.Get("time"), 10, 64)
if u.View.User == (common.Address{}) {
return u.View.FromValues(values)
if u.Feed.User == (common.Address{}) {
return u.Feed.FromValues(values)
}
return nil
}
@ -119,5 +119,5 @@ func (u *ID) FromValues(values Values) error {
func (u *ID) AppendValues(values Values) {
values.Set("level", fmt.Sprintf("%d", u.Epoch.Level))
values.Set("time", fmt.Sprintf("%d", u.Epoch.Time))
u.View.AppendValues(values)
u.Feed.AppendValues(values)
}

10
swarm/storage/mru/id_test.go → swarm/storage/feed/id_test.go
View File

@ -1,21 +1,21 @@
package mru
package feed
import (
"testing"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
func getTestID() *ID {
return &ID{
View: *getTestView(),
Feed: *getTestFeed(),
Epoch: lookup.GetFirstEpoch(1000),
}
}
func TestIDAddr(t *testing.T) {
ul := getTestID()
updateAddr := ul.Addr()
id := getTestID()
updateAddr := id.Addr()
compareByteSliceToExpectedHex(t, "updateAddr", updateAddr, "0x8b24583ec293e085f4c78aaee66d1bc5abfb8b4233304d14a349afa57af2a783")
}

0
swarm/storage/mru/lookup/epoch.go → swarm/storage/feed/lookup/epoch.go
View File

2
swarm/storage/mru/lookup/epoch_test.go → swarm/storage/feed/lookup/epoch_test.go
View File

@ -3,7 +3,7 @@ package lookup_test
import (
"testing"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
func TestMarshallers(t *testing.T) {

2
swarm/storage/mru/lookup/lookup.go → swarm/storage/feed/lookup/lookup.go
View File

@ -15,7 +15,7 @@
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
Package lookup defines resource lookup algorithms and provides tools to place updates
Package lookup defines feed lookup algorithms and provides tools to place updates
so they can be found
*/
package lookup

2
swarm/storage/mru/lookup/lookup_test.go → swarm/storage/feed/lookup/lookup_test.go
View File

@ -22,7 +22,7 @@ import (
"testing"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
type Data struct {

22
swarm/storage/mru/query.go → swarm/storage/feed/query.go
View File

@ -14,20 +14,20 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"fmt"
"strconv"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
// Query is used to specify constraints when performing an update lookup
// TimeLimit indicates an upper bound for the search. Set to 0 for "now"
type Query struct {
View
Feed
Hint lookup.Epoch
TimeLimit uint64
}
@ -41,8 +41,8 @@ func (q *Query) FromValues(values Values) error {
level, _ := strconv.ParseUint(values.Get("hint.level"), 10, 32)
q.Hint.Level = uint8(level)
q.Hint.Time, _ = strconv.ParseUint(values.Get("hint.time"), 10, 64)
if q.View.User == (common.Address{}) {
return q.View.FromValues(values)
if q.Feed.User == (common.Address{}) {
return q.Feed.FromValues(values)
}
return nil
}
@ -59,20 +59,20 @@ func (q *Query) AppendValues(values Values) {
if q.Hint.Time != 0 {
values.Set("hint.time", fmt.Sprintf("%d", q.Hint.Time))
}
q.View.AppendValues(values)
q.Feed.AppendValues(values)
}
// NewQuery constructs an Query structure to find updates on or before `time`
// if time == 0, the latest update will be looked up
func NewQuery(view *View, time uint64, hint lookup.Epoch) *Query {
func NewQuery(feed *Feed, time uint64, hint lookup.Epoch) *Query {
return &Query{
TimeLimit: time,
View: *view,
Feed: *feed,
Hint: hint,
}
}
// NewQueryLatest generates lookup parameters that look for the latest version of a resource
func NewQueryLatest(view *View, hint lookup.Epoch) *Query {
return NewQuery(view, 0, hint)
// NewQueryLatest generates lookup parameters that look for the latest update to a feed
func NewQueryLatest(feed *Feed, hint lookup.Epoch) *Query {
return NewQuery(feed, 0, hint)
}

8
swarm/storage/mru/query_test.go → swarm/storage/feed/query_test.go
View File

@ -14,18 +14,18 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"testing"
)
func getTestQuery() *Query {
ul := getTestID()
id := getTestID()
return &Query{
TimeLimit: 5000,
View: ul.View,
Hint: ul.Epoch,
Feed: id.Feed,
Hint: id.Epoch,
}
}

50
swarm/storage/mru/request.go → swarm/storage/feed/request.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"bytes"
@ -24,15 +24,15 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
// Request represents an update and/or resource create message
// Request represents a request to sign or signed feed update message
type Request struct {
ResourceUpdate // actual content that will be put on the chunk, less signature
Signature *Signature
idAddr storage.Address // cached chunk address for the update (not serialized, for internal use)
binaryData []byte // cached serialized data (does not get serialized again!, for efficiency/internal use)
Update // actual content that will be put on the chunk, less signature
Signature *Signature
idAddr storage.Address // cached chunk address for the update (not serialized, for internal use)
binaryData []byte // cached serialized data (does not get serialized again!, for efficiency/internal use)
}
// updateRequestJSON represents a JSON-serialized UpdateRequest
@ -44,11 +44,11 @@ type updateRequestJSON struct {
}
// Request layout
// resourceUpdate bytes
// Update bytes
// SignatureLength bytes
const minimumSignedUpdateLength = minimumUpdateDataLength + signatureLength
// NewFirstRequest returns a ready to sign request to publish a first update
// NewFirstRequest returns a ready to sign request to publish a first feed update
func NewFirstRequest(topic Topic) *Request {
request := new(Request)
@ -56,13 +56,13 @@ func NewFirstRequest(topic Topic) *Request {
// get the current time
now := TimestampProvider.Now().Time
request.Epoch = lookup.GetFirstEpoch(now)
request.View.Topic = topic
request.Feed.Topic = topic
request.Header.Version = ProtocolVersion
return request
}
// SetData stores the payload data the resource will be updated with
// SetData stores the payload data the feed update will be updated with
func (r *Request) SetData(data []byte) {
r.data = data
r.Signature = nil
@ -73,7 +73,7 @@ func (r *Request) IsUpdate() bool {
return r.Signature != nil
}
// Verify checks that signatures are valid and that the signer owns the resource to be updated
// Verify checks that signatures are valid
func (r *Request) Verify() (err error) {
if len(r.data) == 0 {
return NewError(ErrInvalidValue, "Update does not contain data")
@ -88,7 +88,7 @@ func (r *Request) Verify() (err error) {
}
// get the address of the signer (which also checks that it's a valid signature)
r.View.User, err = getUserAddr(digest, *r.Signature)
r.Feed.User, err = getUserAddr(digest, *r.Signature)
if err != nil {
return err
}
@ -103,9 +103,9 @@ func (r *Request) Verify() (err error) {
return nil
}
// Sign executes the signature to validate the resource
// Sign executes the signature to validate the update message
func (r *Request) Sign(signer Signer) error {
r.View.User = signer.Address()
r.Feed.User = signer.Address()
r.binaryData = nil //invalidate serialized data
digest, err := r.GetDigest() // computes digest and serializes into .binaryData
if err != nil {
@ -133,16 +133,16 @@ func (r *Request) Sign(signer Signer) error {
return nil
}
// GetDigest creates the resource update digest used in signatures
// GetDigest creates the feed update digest used in signatures
// the serialized payload is cached in .binaryData
func (r *Request) GetDigest() (result common.Hash, err error) {
hasher := hashPool.Get().(hash.Hash)
defer hashPool.Put(hasher)
hasher.Reset()
dataLength := r.ResourceUpdate.binaryLength()
dataLength := r.Update.binaryLength()
if r.binaryData == nil {
r.binaryData = make([]byte, dataLength+signatureLength)
if err := r.ResourceUpdate.binaryPut(r.binaryData[:dataLength]); err != nil {
if err := r.Update.binaryPut(r.binaryData[:dataLength]); err != nil {
return result, err
}
}
@ -161,10 +161,10 @@ func (r *Request) toChunk() (storage.Chunk, error) {
return nil, NewError(ErrInvalidSignature, "toChunk called without a valid signature or payload data. Call .Sign() first.")
}
resourceUpdateLength := r.ResourceUpdate.binaryLength()
updateLength := r.Update.binaryLength()
// signature is the last item in the chunk data
copy(r.binaryData[resourceUpdateLength:], r.Signature[:])
copy(r.binaryData[updateLength:], r.Signature[:])
chunk := storage.NewChunk(r.idAddr, r.binaryData)
return chunk, nil
@ -174,14 +174,14 @@ func (r *Request) toChunk() (storage.Chunk, error) {
func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error {
// for update chunk layout see Request definition
//deserialize the resource update portion
if err := r.ResourceUpdate.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil {
//deserialize the feed update portion
if err := r.Update.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil {
return err
}
// Extract the signature
var signature *Signature
cursor := r.ResourceUpdate.binaryLength()
cursor := r.Update.binaryLength()
sigdata := chunkdata[cursor : cursor+signatureLength]
if len(sigdata) > 0 {
signature = &Signature{}
@ -209,7 +209,7 @@ func (r *Request) FromValues(values Values, data []byte) error {
r.Signature = new(Signature)
copy(r.Signature[:], signatureBytes)
}
err = r.ResourceUpdate.FromValues(values, data)
err = r.Update.FromValues(values, data)
if err != nil {
return err
}
@ -223,7 +223,7 @@ func (r *Request) AppendValues(values Values) []byte {
if r.Signature != nil {
values.Set("signature", hexutil.Encode(r.Signature[:]))
}
return r.ResourceUpdate.AppendValues(values)
return r.Update.AppendValues(values)
}
// fromJSON takes an update request JSON and populates an UpdateRequest

58
swarm/storage/mru/request_test.go → swarm/storage/feed/request_test.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"bytes"
@ -26,7 +26,7 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
)
func areEqualJSON(s1, s2 string) (bool, error) {
@ -47,51 +47,49 @@ func areEqualJSON(s1, s2 string) (bool, error) {
}
// TestEncodingDecodingUpdateRequests ensures that requests are serialized properly
// while also checking cryptographically that only the owner of a resource can update it.
// while also checking cryptographically that only the owner of a feed can update it.
func TestEncodingDecodingUpdateRequests(t *testing.T) {
charlie := newCharlieSigner() //Charlie
bob := newBobSigner() //Bob
// Create a resource to our good guy Charlie's name
topic, _ := NewTopic("a good resource name", nil)
createRequest := NewFirstRequest(topic)
createRequest.User = charlie.Address()
// Create a feed to our good guy Charlie's name
topic, _ := NewTopic("a good topic name", nil)
firstRequest := NewFirstRequest(topic)
firstRequest.User = charlie.Address()
// We now encode the create message to simulate we send it over the wire
messageRawData, err := createRequest.MarshalJSON()
messageRawData, err := firstRequest.MarshalJSON()
if err != nil {
t.Fatalf("Error encoding create resource request: %s", err)
t.Fatalf("Error encoding first feed update request: %s", err)
}
// ... the message arrives and is decoded...
var recoveredCreateRequest Request
if err := recoveredCreateRequest.UnmarshalJSON(messageRawData); err != nil {
t.Fatalf("Error decoding create resource request: %s", err)
var recoveredFirstRequest Request
if err := recoveredFirstRequest.UnmarshalJSON(messageRawData); err != nil {
t.Fatalf("Error decoding first feed update request: %s", err)
}
// ... but verification should fail because it is not signed!
if err := recoveredCreateRequest.Verify(); err == nil {
if err := recoveredFirstRequest.Verify(); err == nil {
t.Fatal("Expected Verify to fail since the message is not signed")
}
// We now assume that the resource was created and propagated. With rootAddr we can retrieve the resource metadata
// and recover the information above. To sign an update, we need the rootAddr and the metaHash to construct
// proof of ownership
// We now assume that the feed ypdate was created and propagated.
const expectedSignature = "0x32c2d2c7224e24e4d3ae6a10595fc6e945f1b3ecdf548a04d8247c240a50c9240076aa7730abad6c8a46dfea00cfb8f43b6211f02db5c4cc5ed8584cb0212a4d00"
const expectedJSON = `{"view":{"topic":"0x6120676f6f64207265736f75726365206e616d65000000000000000000000000","user":"0x876a8936a7cd0b79ef0735ad0896c1afe278781c"},"epoch":{"time":1000,"level":1},"protocolVersion":0,"data":"0x5468697320686f75722773207570646174653a20537761726d2039392e3020686173206265656e2072656c656173656421"}`
const expectedSignature = "0x7235b27a68372ddebcf78eba48543fa460864b0b0e99cb533fcd3664820e603312d29426dd00fb39628f5299480a69bf6e462838d78de49ce0704c754c9deb2601"
const expectedJSON = `{"feed":{"topic":"0x6120676f6f6420746f706963206e616d65000000000000000000000000000000","user":"0x876a8936a7cd0b79ef0735ad0896c1afe278781c"},"epoch":{"time":1000,"level":1},"protocolVersion":0,"data":"0x5468697320686f75722773207570646174653a20537761726d2039392e3020686173206265656e2072656c656173656421"}`
//Put together an unsigned update request that we will serialize to send it to the signer.
data := []byte("This hour's update: Swarm 99.0 has been released!")
request := &Request{
ResourceUpdate: ResourceUpdate{
Update: Update{
ID: ID{
Epoch: lookup.Epoch{
Time: 1000,
Level: 1,
},
View: createRequest.ResourceUpdate.View,
Feed: firstRequest.Update.Feed,
},
data: data,
},
@ -138,7 +136,7 @@ func TestEncodingDecodingUpdateRequests(t *testing.T) {
t.Fatal("Expected DecodeUpdateRequest to fail when trying to interpret a corrupt message with an invalid signature")
}
// Now imagine Bob wants to create an update of his own about the same resource,
// Now imagine Bob wants to create an update of his own about the same feed,
// signing a message with his private key
if err := request.Sign(bob); err != nil {
t.Fatalf("Error signing: %s", err)
@ -191,7 +189,7 @@ func TestEncodingDecodingUpdateRequests(t *testing.T) {
func getTestRequest() *Request {
return &Request{
ResourceUpdate: *getTestResourceUpdate(),
Update: *getTestFeedUpdate(),
}
}
@ -230,7 +228,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) {
var recovered Request
recovered.fromChunk(chunk.Address(), chunk.Data())
if !reflect.DeepEqual(recovered, r) {
t.Fatal("Expected recovered SignedResource update to equal the original one")
t.Fatal("Expected recovered feed update request to equal the original one")
}
}
@ -250,7 +248,7 @@ func TestReverse(t *testing.T) {
// signer containing private key
signer := newAliceSigner()
// set up rpc and create resourcehandler
// set up rpc and create feeds handler
_, _, teardownTest, err := setupTest(timeProvider, signer)
if err != nil {
t.Fatal(err)
@ -258,7 +256,7 @@ func TestReverse(t *testing.T) {
defer teardownTest()
topic, _ := NewTopic("Cervantes quotes", nil)
view := View{
fd := Feed{
Topic: topic,
User: signer.Address(),
}
@ -266,7 +264,7 @@ func TestReverse(t *testing.T) {
data := []byte("Donde una puerta se cierra, otra se abre")
request := new(Request)
request.View = view
request.Feed = fd
request.Epoch = epoch
request.data = data
@ -291,15 +289,15 @@ func TestReverse(t *testing.T) {
if err != nil {
t.Fatal(err)
}
recoveredaddress, err := getUserAddr(checkdigest, *checkUpdate.Signature)
recoveredAddr, err := getUserAddr(checkdigest, *checkUpdate.Signature)
if err != nil {
t.Fatalf("Retrieve address from signature fail: %v", err)
}
originaladdress := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
originalAddr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
// check that the metadata retrieved from the chunk matches what we gave it
if recoveredaddress != originaladdress {
t.Fatalf("addresses dont match: %x != %x", originaladdress, recoveredaddress)
if recoveredAddr != originalAddr {
t.Fatalf("addresses dont match: %x != %x", originalAddr, recoveredAddr)
}
if !bytes.Equal(key[:], chunk.Address()[:]) {

6
swarm/storage/mru/resource_sign.go → swarm/storage/feed/sign.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"crypto/ecdsa"
@ -28,7 +28,7 @@ const signatureLength = 65
// Signature is an alias for a static byte array with the size of a signature
type Signature [signatureLength]byte
// Signer signs Mutable Resource update payloads
// Signer signs feed update payloads
type Signer interface {
Sign(common.Hash) (Signature, error)
Address() common.Address
@ -65,7 +65,7 @@ func (s *GenericSigner) Address() common.Address {
return s.address
}
// getUserAddr extracts the address of the resource update signer
// getUserAddr extracts the address of the feed update signer
func getUserAddr(digest common.Hash, signature Signature) (common.Address, error) {
pub, err := crypto.SigToPub(digest.Bytes(), signature[:])
if err != nil {

14
swarm/storage/mru/testutil.go → swarm/storage/feed/testutil.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"context"
@ -27,7 +27,7 @@ import (
)
const (
testDbDirName = "mru"
testDbDirName = "feeds"
)
type TestHandler struct {
@ -52,20 +52,20 @@ func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetF
// NewTestHandler creates Handler object to be used for testing purposes.
func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) {
path := filepath.Join(datadir, testDbDirName)
rh := NewHandler(params)
fh := NewHandler(params)
localstoreparams := storage.NewDefaultLocalStoreParams()
localstoreparams.Init(path)
localStore, err := storage.NewLocalStore(localstoreparams, nil)
if err != nil {
return nil, fmt.Errorf("localstore create fail, path %s: %v", path, err)
}
localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(resourceHashAlgorithm)))
localStore.Validators = append(localStore.Validators, rh)
localStore.Validators = append(localStore.Validators, storage.NewContentAddressValidator(storage.MakeHashFunc(feedsHashAlgorithm)))
localStore.Validators = append(localStore.Validators, fh)
netStore, err := storage.NewNetStore(localStore, nil)
if err != nil {
return nil, err
}
netStore.NewNetFetcherFunc = newFakeNetFetcher
rh.SetStore(netStore)
return &TestHandler{rh}, nil
fh.SetStore(netStore)
return &TestHandler{fh}, nil
}

4
swarm/storage/mru/timestampprovider.go → swarm/storage/feed/timestampprovider.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"encoding/binary"
@ -22,7 +22,7 @@ import (
"time"
)
// TimestampProvider sets the time source of the mru package
// TimestampProvider sets the time source of the feeds package
var TimestampProvider timestampProvider = NewDefaultTimestampProvider()
// Timestamp encodes a point in time as a Unix epoch

6
swarm/storage/mru/topic.go → swarm/storage/feed/topic.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"bytes"
@ -29,7 +29,7 @@ import (
// TopicLength establishes the max length of a topic string
const TopicLength = storage.AddressLength
// Topic represents what a resource talks about
// Topic represents what a feed is about
type Topic [TopicLength]byte
// ErrTopicTooLong is returned when creating a topic with a name/related content too long
@ -74,7 +74,7 @@ func (t *Topic) FromHex(hex string) error {
return nil
}
// Name will try to extract the resource name out of the topic
// Name will try to extract the topic name out of the Topic
func (t *Topic) Name(relatedContent []byte) string {
nameBytes := *t
if relatedContent != nil {

2
swarm/storage/mru/topic_test.go → swarm/storage/feed/topic_test.go
View File

@ -1,4 +1,4 @@
package mru
package feed
import (
"testing"

26
swarm/storage/mru/update.go → swarm/storage/feed/update.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"fmt"
@ -34,25 +34,25 @@ type Header struct {
Padding [headerLength - 1]uint8 // reserved for future use
}
// ResourceUpdate encapsulates the information sent as part of a resource update
type ResourceUpdate struct {
// Update encapsulates the information sent as part of a feed update
type Update struct {
Header Header //
ID // Resource update identifying information
ID // Feed Update identifying information
data []byte // actual data payload
}
const minimumUpdateDataLength = idLength + headerLength + 1
const maxUpdateDataLength = chunk.DefaultSize - signatureLength - idLength - headerLength
// binaryPut serializes the resource update information into the given slice
func (r *ResourceUpdate) binaryPut(serializedData []byte) error {
// binaryPut serializes the feed update information into the given slice
func (r *Update) binaryPut(serializedData []byte) error {
datalength := len(r.data)
if datalength == 0 {
return NewError(ErrInvalidValue, "cannot update a resource with no data")
return NewError(ErrInvalidValue, "a feed update must contain data")
}
if datalength > maxUpdateDataLength {
return NewErrorf(ErrInvalidValue, "data is too big (length=%d). Max length=%d", datalength, maxUpdateDataLength)
return NewErrorf(ErrInvalidValue, "feed update data is too big (length=%d). Max length=%d", datalength, maxUpdateDataLength)
}
if len(serializedData) != r.binaryLength() {
@ -79,14 +79,14 @@ func (r *ResourceUpdate) binaryPut(serializedData []byte) error {
}
// binaryLength returns the expected number of bytes this structure will take to encode
func (r *ResourceUpdate) binaryLength() int {
func (r *Update) binaryLength() int {
return idLength + headerLength + len(r.data)
}
// binaryGet populates this instance from the information contained in the passed byte slice
func (r *ResourceUpdate) binaryGet(serializedData []byte) error {
func (r *Update) binaryGet(serializedData []byte) error {
if len(serializedData) < minimumUpdateDataLength {
return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a resource update chunk", minimumUpdateDataLength)
return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a feed update chunk", minimumUpdateDataLength)
}
dataLength := len(serializedData) - idLength - headerLength
// at this point we can be satisfied that we have the correct data length to read
@ -116,7 +116,7 @@ func (r *ResourceUpdate) binaryGet(serializedData []byte) error {
// FromValues deserializes this instance from a string key-value store
// useful to parse query strings
func (r *ResourceUpdate) FromValues(values Values, data []byte) error {
func (r *Update) FromValues(values Values, data []byte) error {
r.data = data
version, _ := strconv.ParseUint(values.Get("protocolVersion"), 10, 32)
r.Header.Version = uint8(version)
@ -125,7 +125,7 @@ func (r *ResourceUpdate) FromValues(values Values, data []byte) error {
// AppendValues serializes this structure into the provided string key-value store
// useful to build query strings
func (r *ResourceUpdate) AppendValues(values Values) []byte {
func (r *Update) AppendValues(values Values) []byte {
r.ID.AppendValues(values)
values.Set("protocolVersion", fmt.Sprintf("%d", r.Header.Version))
return r.data

20
swarm/storage/mru/update_test.go → swarm/storage/feed/update_test.go
View File

@ -14,37 +14,37 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package mru
package feed
import (
"testing"
)
func getTestResourceUpdate() *ResourceUpdate {
return &ResourceUpdate{
func getTestFeedUpdate() *Update {
return &Update{
ID: *getTestID(),
data: []byte("El que lee mucho y anda mucho, ve mucho y sabe mucho"),
}
}
func TestResourceUpdateSerializer(t *testing.T) {
testBinarySerializerRecovery(t, getTestResourceUpdate(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019456c20717565206c6565206d7563686f207920616e6461206d7563686f2c207665206d7563686f20792073616265206d7563686f")
func TestUpdateSerializer(t *testing.T) {
testBinarySerializerRecovery(t, getTestFeedUpdate(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019456c20717565206c6565206d7563686f207920616e6461206d7563686f2c207665206d7563686f20792073616265206d7563686f")
}
func TestResourceUpdateLengthCheck(t *testing.T) {
testBinarySerializerLengthCheck(t, getTestResourceUpdate())
func TestUpdateLengthCheck(t *testing.T) {
testBinarySerializerLengthCheck(t, getTestFeedUpdate())
// Test fail if update is too big
update := getTestResourceUpdate()
update := getTestFeedUpdate()
update.data = make([]byte, maxUpdateDataLength+100)
serialized := make([]byte, update.binaryLength())
if err := update.binaryPut(serialized); err == nil {
t.Fatal("Expected resourceUpdate.binaryPut to fail since update is too big")
t.Fatal("Expected update.binaryPut to fail since update is too big")
}
// test fail if data is empty or nil
update.data = nil
serialized = make([]byte, update.binaryLength())
if err := update.binaryPut(serialized); err == nil {
t.Fatal("Expected resourceUpdate.binaryPut to fail since data is empty")
t.Fatal("Expected update.binaryPut to fail since data is empty")
}
}

4
swarm/storage/localstore_test.go
View File

@ -30,8 +30,8 @@ var (
)
// tests that the content address validator correctly checks the data
// tests that resource update chunks are passed through content address validator
// the test checking the resouce update validator internal correctness is found in resource_test.go
// tests that feed update chunks are passed through content address validator
// the test checking the resouce update validator internal correctness is found in storage/feeds/handler_test.go
func TestValidator(t *testing.T) {
// set up localstore
datadir, err := ioutil.TempDir("", "storage-testvalidator")

44
swarm/storage/mru/doc.go
View File

@ -1,44 +0,0 @@
/*
Package mru defines Mutable resource updates.
A Mutable Resource is an entity which allows updates to a resource
without resorting to ENS on each update.
The update scheme is built on swarm chunks with chunk keys following
a predictable, versionable pattern.
A Resource is tied to a unique identifier that is deterministically generated out of
the chosen topic.
A Resource View is defined as a specific user's point of view about a particular resource.
Thus, a View is a Topic + the user's address (userAddr)
Actual data updates are also made in the form of swarm chunks. The keys
of the updates are the hash of a concatenation of properties as follows:
updateAddr = H(View, Epoch ID)
where H is the SHA3 hash function
View is the combination of Topic and the user address
Epoch ID is a time slot. See the lookup package for more information.
A user looking up a resource would only need to know the View in order to
another user's updates
The resource update data is:
resourcedata = View|Epoch|data
the full update data that goes in the chunk payload is:
resourcedata|sign(resourcedata)
Structure Summary:
Request: Resource update with signature
ResourceUpdate: headers + data
Header: Protocol version and reserved for future use placeholders
ID: Information about how to locate a specific update
View: Author of the update and what is updating
Topic: Item that the updates are about
User: User who updates the resource
Epoch: time slot where the update is stored
*/
package mru