gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

Browse Source 
* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate
This commit is contained in:
Javier Peletier
2018-09-28 12:07:17 +02:00
committed by Martin Holst Swende
parent 0da3b17a11
commit 2c110c81ee
44 changed files with 3337 additions and 3022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
swarm/api/client/client.go
View File

@@ -28,6 +28,7 @@ import (
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"regexp"
@@ -595,13 +596,16 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error)
return string(data), nil
}
// ErrNoResourceUpdatesFound is returned when Swarm cannot find updates of the given resource
var ErrNoResourceUpdatesFound = errors.New("No updates found for this resource")
// CreateResource creates a Mutable Resource with the given name and frequency, initializing it with the provided
// data. Data is interpreted as multihash or not depending on the multihash parameter.
// startTime=0 means "now"
// Returns the resulting Mutable Resource manifest address that you can use to include in an ENS Resolver (setContent)
// or reference future updates (Client.UpdateResource)
func (c *Client) CreateResource(request *mru.Request) (string, error) {
responseStream, err := c.updateResource(request)
responseStream, err := c.updateResource(request, true)
if err != nil {
return "", err
}
@@ -621,17 +625,24 @@ func (c *Client) CreateResource(request *mru.Request) (string, error) {
// UpdateResource allows you to set a new version of your content
func (c *Client) UpdateResource(request *mru.Request) error {
_, err := c.updateResource(request)
_, err := c.updateResource(request, false)
return err
}
func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) {
body, err := request.MarshalJSON()
func (c *Client) updateResource(request *mru.Request, createManifest bool) (io.ReadCloser, error) {
URL, err := url.Parse(c.Gateway)
if err != nil {
return nil, err
}
URL.Path = "/bzz-resource:/"
values := URL.Query()
body := request.AppendValues(values)
if createManifest {
values.Set("manifest", "1")
}
URL.RawQuery = values.Encode()
req, err := http.NewRequest("POST", c.Gateway+"/bzz-resource:/", bytes.NewBuffer(body))
req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body))
if err != nil {
return nil, err
}
@@ -642,28 +653,61 @@ func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) {
}
return res.Body, nil
}
// GetResource returns a byte stream with the raw content of the resource
// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
// points to that address
func (c *Client) GetResource(manifestAddressOrDomain string) (io.ReadCloser, error) {
func (c *Client) GetResource(query *mru.Query, manifestAddressOrDomain string) (io.ReadCloser, error) {
return c.getResource(query, manifestAddressOrDomain, false)
}
res, err := http.Get(c.Gateway + "/bzz-resource:/" + manifestAddressOrDomain)
// getResource returns a byte stream with the raw content of the resource
// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
// points to that address
// meta set to true will instruct the node return resource metainformation instead
func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) {
URL, err := url.Parse(c.Gateway)
if err != nil {
return nil, err
}
URL.Path = "/bzz-resource:/" + manifestAddressOrDomain
values := URL.Query()
if query != nil {
query.AppendValues(values) //adds query parameters
}
if meta {
values.Set("meta", "1")
}
URL.RawQuery = values.Encode()
res, err := http.Get(URL.String())
if err != nil {
return nil, err
}
return res.Body, nil
if res.StatusCode != http.StatusOK {
if res.StatusCode == http.StatusNotFound {
return nil, ErrNoResourceUpdatesFound
}
errorMessageBytes, err := ioutil.ReadAll(res.Body)
var errorMessage string
if err != nil {
errorMessage = "cannot retrieve error message: " + err.Error()
} else {
errorMessage = string(errorMessageBytes)
}
return nil, fmt.Errorf("Error retrieving resource: %s", errorMessage)
}
return res.Body, nil
}
// GetResourceMetadata returns a structure that describes the Mutable Resource
// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
// points to that address
func (c *Client) GetResourceMetadata(manifestAddressOrDomain string) (*mru.Request, error) {
func (c *Client) GetResourceMetadata(query *mru.Query, manifestAddressOrDomain string) (*mru.Request, error) {
responseStream, err := c.GetResource(manifestAddressOrDomain + "/meta")
responseStream, err := c.getResource(query, manifestAddressOrDomain, true)
if err != nil {
return nil, err
}

77
swarm/api/client/client_test.go
View File

@@ -25,6 +25,8 @@ import (
"sort"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/api"
@@ -391,19 +393,12 @@ func TestClientCreateResourceMultihash(t *testing.T) {
s := common.FromHex(swarmHash)
mh := multihash.ToMultihash(s)
// our mutable resource "name"
resourceName := "foo.eth"
// our mutable resource topic
topic, _ := mru.NewTopic("foo.eth", nil)
createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: resourceName,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
if err != nil {
t.Fatal(err)
}
createRequest.SetData(mh, true)
createRequest := mru.NewFirstRequest(topic)
createRequest.SetData(mh)
if err := createRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
@@ -414,12 +409,18 @@ func TestClientCreateResourceMultihash(t *testing.T) {
t.Fatalf("Error creating resource: %s", err)
}
correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
if resourceManifestHash != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
}
reader, err := client.GetResource(correctManifestAddrHex)
// Check we get a not found error when trying to get the resource with a made-up manifest
_, err = client.GetResource(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if err != ErrNoResourceUpdatesFound {
t.Fatalf("Expected to receive ErrNoResourceUpdatesFound error. Got: %s", err)
}
reader, err := client.GetResource(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
@@ -447,30 +448,22 @@ func TestClientCreateUpdateResource(t *testing.T) {
databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...")
// our mutable resource name
resourceName := "El Quijote"
topic, _ := mru.NewTopic("El Quijote", nil)
createRequest := mru.NewFirstRequest(topic)
createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: resourceName,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
if err != nil {
t.Fatal(err)
}
createRequest.SetData(databytes, false)
createRequest.SetData(databytes)
if err := createRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
resourceManifestHash, err := client.CreateResource(createRequest)
correctManifestAddrHex := "cc7904c17b49f9679e2d8006fe25e87e3f5c2072c2b49cab50f15e544471b30a"
correctManifestAddrHex := "fcb8e75f53e480e197c083ad1976d265674d0ce776f2bf359c09c413fb5230b8"
if resourceManifestHash != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
}
reader, err := client.GetResource(correctManifestAddrHex)
reader, err := client.GetResource(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
@@ -486,12 +479,12 @@ func TestClientCreateUpdateResource(t *testing.T) {
// define different data
databytes = []byte("... no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero ...")
updateRequest, err := client.GetResourceMetadata(correctManifestAddrHex)
updateRequest, err := client.GetResourceMetadata(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving update request template: %s", err)
}
updateRequest.SetData(databytes, false)
updateRequest.SetData(databytes)
if err := updateRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
@@ -500,7 +493,7 @@ func TestClientCreateUpdateResource(t *testing.T) {
t.Fatalf("Error updating resource: %s", err)
}
reader, err = client.GetResource(correctManifestAddrHex)
reader, err = client.GetResource(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
@@ -513,4 +506,24 @@ func TestClientCreateUpdateResource(t *testing.T) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
// now try retrieving resource without a manifest
view := &mru.View{
Topic: topic,
User: signer.Address(),
}
lookupParams := mru.NewQueryLatest(view, lookup.NoClue)
reader, err = client.GetResource(lookupParams, "")
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
defer reader.Close()
gotData, err = ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, gotData) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
}