gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

Browse Source 
* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate
This commit is contained in:
Javier Peletier
2018-09-28 12:07:17 +02:00
committed by Martin Holst Swende
parent 0da3b17a11
commit 2c110c81ee
44 changed files with 3337 additions and 3022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
swarm/api/api.go
View File

@ -29,6 +29,8 @@ import (
"path"
"strings"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"bytes"
"mime"
"path/filepath"
@ -401,77 +403,54 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
// we need to do some extra work if this is a mutable resource manifest
if entry.ContentType == ResourceContentType {
// get the resource rootAddr
log.Trace("resource type", "menifestAddr", manifestAddr, "hash", entry.Hash)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rootAddr := storage.Address(common.FromHex(entry.Hash))
rsrc, err := a.resource.Load(ctx, rootAddr)
if entry.ResourceView == nil {
return reader, mimeType, status, nil, fmt.Errorf("Cannot decode ResourceView in manifest")
}
_, err := a.resource.Lookup(ctx, mru.NewQueryLatest(entry.ResourceView, lookup.NoClue))
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Debug(fmt.Sprintf("get resource content error: %v", err))
return reader, mimeType, status, nil, err
}
// use this key to retrieve the latest update
params := mru.LookupLatest(rootAddr)
rsrc, err = a.resource.Lookup(ctx, params)
// get the data of the update
_, rsrcData, err := a.resource.GetContent(entry.ResourceView)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Debug(fmt.Sprintf("get resource content error: %v", err))
log.Warn(fmt.Sprintf("get resource content error: %v", err))
return reader, mimeType, status, nil, err
}
// if it's multihash, we will transparently serve the content this multihash points to
// \TODO this resolve is rather expensive all in all, review to see if it can be achieved cheaper
if rsrc.Multihash() {
// extract multihash
decodedMultihash, err := multihash.FromMultihash(rsrcData)
if err != nil {
apiGetInvalid.Inc(1)
status = http.StatusUnprocessableEntity
log.Warn("invalid resource multihash", "err", err)
return reader, mimeType, status, nil, err
}
manifestAddr = storage.Address(decodedMultihash)
log.Trace("resource is multihash", "key", manifestAddr)
// get the data of the update
_, rsrcData, err := a.resource.GetContent(rootAddr)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("get resource content error: %v", err))
return reader, mimeType, status, nil, err
}
// get the manifest the multihash digest points to
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
return reader, mimeType, status, nil, err
}
// validate that data as multihash
decodedMultihash, err := multihash.FromMultihash(rsrcData)
if err != nil {
apiGetInvalid.Inc(1)
status = http.StatusUnprocessableEntity
log.Warn("invalid resource multihash", "err", err)
return reader, mimeType, status, nil, err
}
manifestAddr = storage.Address(decodedMultihash)
log.Trace("resource is multihash", "key", manifestAddr)
// get the manifest the multihash digest points to
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
return reader, mimeType, status, nil, err
}
// finally, get the manifest entry
// it will always be the entry on path ""
entry, _ = trie.getEntry(path)
if entry == nil {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path)
return reader, mimeType, status, nil, err
}
} else {
// data is returned verbatim since it's not a multihash
return rsrc, "application/octet-stream", http.StatusOK, nil, nil
// finally, get the manifest entry
// it will always be the entry on path ""
entry, _ = trie.getEntry(path)
if entry == nil {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path)
return reader, mimeType, status, nil, err
}
}
@ -966,37 +945,27 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver
}
// ResourceLookup finds mutable resource updates at specific periods and versions
func (a *API) ResourceLookup(ctx context.Context, params *mru.LookupParams) (string, []byte, error) {
var err error
rsrc, err := a.resource.Load(ctx, params.RootAddr())
func (a *API) ResourceLookup(ctx context.Context, query *mru.Query) ([]byte, error) {
_, err := a.resource.Lookup(ctx, query)
if err != nil {
return "", nil, err
}
_, err = a.resource.Lookup(ctx, params)
if err != nil {
return "", nil, err
return nil, err
}
var data []byte
_, data, err = a.resource.GetContent(params.RootAddr())
_, data, err = a.resource.GetContent(&query.View)
if err != nil {
return "", nil, err
return nil, err
}
return rsrc.Name(), data, nil
}
// Create Mutable resource
func (a *API) ResourceCreate(ctx context.Context, request *mru.Request) error {
return a.resource.New(ctx, request)
return data, nil
}
// ResourceNewRequest creates a Request object to update a specific mutable resource
func (a *API) ResourceNewRequest(ctx context.Context, rootAddr storage.Address) (*mru.Request, error) {
return a.resource.NewUpdateRequest(ctx, rootAddr)
func (a *API) ResourceNewRequest(ctx context.Context, view *mru.View) (*mru.Request, error) {
return a.resource.NewRequest(ctx, view)
}
// ResourceUpdate updates a Mutable Resource with arbitrary data.
// Upon retrieval the update will be retrieved verbatim as bytes.
func (a *API) ResourceUpdate(ctx context.Context, request *mru.SignedResourceUpdate) (storage.Address, error) {
func (a *API) ResourceUpdate(ctx context.Context, request *mru.Request) (storage.Address, error) {
return a.resource.Update(ctx, request)
}
@ -1005,17 +974,62 @@ func (a *API) ResourceHashSize() int {
return a.resource.HashSize
}
// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk.
func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) {
// ErrCannotLoadResourceManifest is returned when looking up a resource manifest fails
var ErrCannotLoadResourceManifest = errors.New("Cannot load resource manifest")
// ErrNotAResourceManifest is returned when the address provided returned something other than a valid manifest
var ErrNotAResourceManifest = errors.New("Not a resource manifest")
// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the Resource's view ID.
func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (*mru.View, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt)
if err != nil {
return nil, fmt.Errorf("cannot load resource manifest: %v", err)
return nil, ErrCannotLoadResourceManifest
}
entry, _ := trie.getEntry("")
if entry.ContentType != ResourceContentType {
return nil, fmt.Errorf("not a resource manifest: %s", addr)
return nil, ErrNotAResourceManifest
}
return storage.Address(common.FromHex(entry.Hash)), nil
return entry.ResourceView, nil
}
// ErrCannotResolveResourceURI is returned when the ENS resolver is not able to translate a name to a resource
var ErrCannotResolveResourceURI = errors.New("Cannot resolve Resource URI")
// ErrCannotResolveResourceView is returned when values provided are not enough or invalid to recreate a
// resource view out of them.
var ErrCannotResolveResourceView = errors.New("Cannot resolve resource view")
// ResolveResourceView attempts to extract View information out of the manifest, if provided
// If not, it attempts to extract the View out of a set of key-value pairs
func (a *API) ResolveResourceView(ctx context.Context, uri *URI, values mru.Values) (*mru.View, error) {
var view *mru.View
var err error
if uri.Addr != "" {
// resolve the content key.
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = a.Resolve(ctx, uri.Addr)
if err != nil {
return nil, ErrCannotResolveResourceURI
}
}
// get the resource view from the manifest
view, err = a.ResolveResourceManifest(ctx, manifestAddr)
if err != nil {
return nil, err
}
log.Debug("handle.get.resource: resolved", "manifestkey", manifestAddr, "view", view.Hex())
} else {
var v mru.View
if err := v.FromValues(values); err != nil {
return nil, ErrCannotResolveResourceView
}
view = &v
}
return view, nil
}

66
swarm/api/client/client.go
View File

@ -28,6 +28,7 @@ import (
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"regexp"
@ -595,13 +596,16 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error)
return string(data), nil
}
// ErrNoResourceUpdatesFound is returned when Swarm cannot find updates of the given resource
var ErrNoResourceUpdatesFound = errors.New("No updates found for this resource")
// CreateResource creates a Mutable Resource with the given name and frequency, initializing it with the provided
// data. Data is interpreted as multihash or not depending on the multihash parameter.
// startTime=0 means "now"
// Returns the resulting Mutable Resource manifest address that you can use to include in an ENS Resolver (setContent)
// or reference future updates (Client.UpdateResource)
func (c *Client) CreateResource(request *mru.Request) (string, error) {
responseStream, err := c.updateResource(request)
responseStream, err := c.updateResource(request, true)
if err != nil {
return "", err
}
@ -621,17 +625,24 @@ func (c *Client) CreateResource(request *mru.Request) (string, error) {
// UpdateResource allows you to set a new version of your content
func (c *Client) UpdateResource(request *mru.Request) error {
_, err := c.updateResource(request)
_, err := c.updateResource(request, false)
return err
}
func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) {
body, err := request.MarshalJSON()
func (c *Client) updateResource(request *mru.Request, createManifest bool) (io.ReadCloser, error) {
URL, err := url.Parse(c.Gateway)
if err != nil {
return nil, err
}
URL.Path = "/bzz-resource:/"
values := URL.Query()
body := request.AppendValues(values)
if createManifest {
values.Set("manifest", "1")
}
URL.RawQuery = values.Encode()
req, err := http.NewRequest("POST", c.Gateway+"/bzz-resource:/", bytes.NewBuffer(body))
req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body))
if err != nil {
return nil, err
}
@ -642,28 +653,61 @@ func (c *Client) updateResource(request *mru.Request) (io.ReadCloser, error) {
}
return res.Body, nil
}
// GetResource returns a byte stream with the raw content of the resource
// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
// points to that address
func (c *Client) GetResource(manifestAddressOrDomain string) (io.ReadCloser, error) {
func (c *Client) GetResource(query *mru.Query, manifestAddressOrDomain string) (io.ReadCloser, error) {
return c.getResource(query, manifestAddressOrDomain, false)
}
res, err := http.Get(c.Gateway + "/bzz-resource:/" + manifestAddressOrDomain)
// getResource returns a byte stream with the raw content of the resource
// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
// points to that address
// meta set to true will instruct the node return resource metainformation instead
func (c *Client) getResource(query *mru.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) {
URL, err := url.Parse(c.Gateway)
if err != nil {
return nil, err
}
URL.Path = "/bzz-resource:/" + manifestAddressOrDomain
values := URL.Query()
if query != nil {
query.AppendValues(values) //adds query parameters
}
if meta {
values.Set("meta", "1")
}
URL.RawQuery = values.Encode()
res, err := http.Get(URL.String())
if err != nil {
return nil, err
}
return res.Body, nil
if res.StatusCode != http.StatusOK {
if res.StatusCode == http.StatusNotFound {
return nil, ErrNoResourceUpdatesFound
}
errorMessageBytes, err := ioutil.ReadAll(res.Body)
var errorMessage string
if err != nil {
errorMessage = "cannot retrieve error message: " + err.Error()
} else {
errorMessage = string(errorMessageBytes)
}
return nil, fmt.Errorf("Error retrieving resource: %s", errorMessage)
}
return res.Body, nil
}
// GetResourceMetadata returns a structure that describes the Mutable Resource
// manifestAddressOrDomain is the address you obtained in CreateResource or an ENS domain whose Resolver
// points to that address
func (c *Client) GetResourceMetadata(manifestAddressOrDomain string) (*mru.Request, error) {
func (c *Client) GetResourceMetadata(query *mru.Query, manifestAddressOrDomain string) (*mru.Request, error) {
responseStream, err := c.GetResource(manifestAddressOrDomain + "/meta")
responseStream, err := c.getResource(query, manifestAddressOrDomain, true)
if err != nil {
return nil, err
}

77
swarm/api/client/client_test.go
View File

@ -25,6 +25,8 @@ import (
"sort"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/api"
@ -391,19 +393,12 @@ func TestClientCreateResourceMultihash(t *testing.T) {
s := common.FromHex(swarmHash)
mh := multihash.ToMultihash(s)
// our mutable resource "name"
resourceName := "foo.eth"
// our mutable resource topic
topic, _ := mru.NewTopic("foo.eth", nil)
createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: resourceName,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
if err != nil {
t.Fatal(err)
}
createRequest.SetData(mh, true)
createRequest := mru.NewFirstRequest(topic)
createRequest.SetData(mh)
if err := createRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
@ -414,12 +409,18 @@ func TestClientCreateResourceMultihash(t *testing.T) {
t.Fatalf("Error creating resource: %s", err)
}
correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
if resourceManifestHash != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
}
reader, err := client.GetResource(correctManifestAddrHex)
// Check we get a not found error when trying to get the resource with a made-up manifest
_, err = client.GetResource(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if err != ErrNoResourceUpdatesFound {
t.Fatalf("Expected to receive ErrNoResourceUpdatesFound error. Got: %s", err)
}
reader, err := client.GetResource(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
@ -447,30 +448,22 @@ func TestClientCreateUpdateResource(t *testing.T) {
databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...")
// our mutable resource name
resourceName := "El Quijote"
topic, _ := mru.NewTopic("El Quijote", nil)
createRequest := mru.NewFirstRequest(topic)
createRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: resourceName,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
if err != nil {
t.Fatal(err)
}
createRequest.SetData(databytes, false)
createRequest.SetData(databytes)
if err := createRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
resourceManifestHash, err := client.CreateResource(createRequest)
correctManifestAddrHex := "cc7904c17b49f9679e2d8006fe25e87e3f5c2072c2b49cab50f15e544471b30a"
correctManifestAddrHex := "fcb8e75f53e480e197c083ad1976d265674d0ce776f2bf359c09c413fb5230b8"
if resourceManifestHash != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, resourceManifestHash)
}
reader, err := client.GetResource(correctManifestAddrHex)
reader, err := client.GetResource(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
@ -486,12 +479,12 @@ func TestClientCreateUpdateResource(t *testing.T) {
// define different data
databytes = []byte("... no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero ...")
updateRequest, err := client.GetResourceMetadata(correctManifestAddrHex)
updateRequest, err := client.GetResourceMetadata(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving update request template: %s", err)
}
updateRequest.SetData(databytes, false)
updateRequest.SetData(databytes)
if err := updateRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
@ -500,7 +493,7 @@ func TestClientCreateUpdateResource(t *testing.T) {
t.Fatalf("Error updating resource: %s", err)
}
reader, err = client.GetResource(correctManifestAddrHex)
reader, err = client.GetResource(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
@ -513,4 +506,24 @@ func TestClientCreateUpdateResource(t *testing.T) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
// now try retrieving resource without a manifest
view := &mru.View{
Topic: topic,
User: signer.Address(),
}
lookupParams := mru.NewQueryLatest(view, lookup.NoClue)
reader, err = client.GetResource(lookupParams, "")
if err != nil {
t.Fatalf("Error retrieving resource: %s", err)
}
defer reader.Close()
gotData, err = ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, gotData) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
}

156
swarm/api/http/server.go
View File

@ -487,6 +487,7 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) {
// The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content
func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.post.resource", "ruid", ruid)
var err error
@ -496,9 +497,24 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) {
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
view, err := s.api.ResolveResourceView(r.Context(), uri, r.URL.Query())
if err != nil { // couldn't parse query string or retrieve manifest
getFail.Inc(1)
httpStatus := http.StatusBadRequest
if err == api.ErrCannotLoadResourceManifest || err == api.ErrCannotResolveResourceURI {
httpStatus = http.StatusNotFound
}
RespondError(w, r, fmt.Sprintf("cannot retrieve resource view: %s", err), httpStatus)
return
}
var updateRequest mru.Request
if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON
RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
updateRequest.View = *view
query := r.URL.Query()
if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters
RespondError(w, r, err.Error(), http.StatusBadRequest)
return
}
@ -510,56 +526,40 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) {
RespondError(w, r, err.Error(), http.StatusForbidden)
return
}
}
if updateRequest.IsNew() {
err = s.api.ResourceCreate(r.Context(), &updateRequest)
if err != nil {
code, err2 := s.translateResourceError(w, r, "resource creation fail", err)
RespondError(w, r, err2.Error(), code)
return
}
}
if updateRequest.IsUpdate() {
_, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate)
_, err = s.api.ResourceUpdate(r.Context(), &updateRequest)
if err != nil {
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
}
// at this point both possible operations (create, update or both) were successful
// so in case it was a new resource, then create a manifest and send it over.
if updateRequest.IsNew() {
if query.Get("manifest") == "1" {
// we create a manifest so we can retrieve the resource with bzz:// later
// this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource
// metadata chunk (rootAddr)
m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex())
// this manifest has a special "resource type" manifest, and saves the
// resource view ID used to retrieve the resource later
m, err := s.api.NewResourceManifest(r.Context(), &updateRequest.View)
if err != nil {
RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
return
}
// the key to the manifest will be passed back to the client
// the client can access the root chunk key directly through its Hash member
// the manifest key should be set as content in the resolver of the ENS name
// \TODO update manifest key automatically in ENS
// the client can access the view directly through its resourceView member
// the manifest key can be set as content in the resolver of the ENS name
outdata, err := json.Marshal(m)
if err != nil {
RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
return
}
fmt.Fprint(w, string(outdata))
w.Header().Add("Content-type", "application/json")
}
w.Header().Add("Content-type", "application/json")
}
// Retrieve mutable resource updates:
// bzz-resource://<id> - get latest update
// bzz-resource://<id>/<n> - get latest update on period n
// bzz-resource://<id>/<n>/<m> - get update version m of period n
// bzz-resource://<id>/?period=n - get latest update on period n
// bzz-resource://<id>/?period=n&version=m - get update version m of period n
// bzz-resource://<id>/meta - get metadata and next version information
// <id> = ens name or hash
// TODO: Enable pass maxPeriod parameter
@ -569,84 +569,44 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
log.Debug("handle.get.resource", "ruid", ruid)
var err error
// resolve the content key.
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
} else {
w.Header().Set("Cache-Control", "max-age=2147483648")
}
// get the root chunk rootAddr from the manifest
rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr)
if err != nil {
view, err := s.api.ResolveResourceView(r.Context(), uri, r.URL.Query())
if err != nil { // couldn't parse query string or retrieve manifest
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound)
httpStatus := http.StatusBadRequest
if err == api.ErrCannotLoadResourceManifest || err == api.ErrCannotResolveResourceURI {
httpStatus = http.StatusNotFound
}
RespondError(w, r, fmt.Sprintf("cannot retrieve resource view: %s", err), httpStatus)
return
}
log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
// determine if the query specifies period and version or it is a metadata query
var params []string
if len(uri.Path) > 0 {
if uri.Path == "meta" {
unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
return
}
rawResponse, err := unsignedUpdateRequest.MarshalJSON()
if err != nil {
RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
return
}
w.Header().Add("Content-type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, string(rawResponse))
if r.URL.Query().Get("meta") == "1" {
unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), view)
if err != nil {
getFail.Inc(1)
RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for view=%s: %s", view.Hex(), err), http.StatusNotFound)
return
}
params = strings.Split(uri.Path, "/")
rawResponse, err := unsignedUpdateRequest.MarshalJSON()
if err != nil {
RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
return
}
w.Header().Add("Content-type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, string(rawResponse))
return
}
var name string
var data []byte
now := time.Now()
switch len(params) {
case 0: // latest only
name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupLatest(rootAddr))
case 2: // specific period and version
var version uint64
var period uint64
version, err = strconv.ParseUint(params[1], 10, 32)
if err != nil {
break
}
period, err = strconv.ParseUint(params[0], 10, 32)
if err != nil {
break
}
name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupVersion(rootAddr, uint32(period), uint32(version)))
case 1: // last version of specific period
var period uint64
period, err = strconv.ParseUint(params[0], 10, 32)
if err != nil {
break
}
name, data, err = s.api.ResourceLookup(r.Context(), mru.LookupLatestVersionInPeriod(rootAddr, uint32(period)))
default: // bogus
err = mru.NewError(storage.ErrInvalidValue, "invalid mutable resource request")
lookupParams := &mru.Query{View: *view}
if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version
RespondError(w, r, fmt.Sprintf("invalid mutable resource request:%s", err), http.StatusBadRequest)
return
}
data, err := s.api.ResourceLookup(r.Context(), lookupParams)
// any error from the switch statement will end up here
if err != nil {
code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err)
@ -655,9 +615,9 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
}
// All ok, serve the retrieved update
log.Debug("Found update", "name", name, "ruid", ruid)
log.Debug("Found update", "view", view.Hex(), "ruid", ruid)
w.Header().Set("Content-Type", "application/octet-stream")
http.ServeContent(w, r, "", now, bytes.NewReader(data))
http.ServeContent(w, r, "", time.Now(), bytes.NewReader(data))
}
func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) {

159
swarm/api/http/server_test.go
View File

@ -30,12 +30,15 @@ import (
"math/big"
"mime/multipart"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@ -121,8 +124,8 @@ func TestBzzResourceMultihash(t *testing.T) {
// add the data our multihash aliased manifest will point to
databytes := "bar"
url := fmt.Sprintf("%s/bzz:/", srv.URL)
resp, err := http.Post(url, "text/plain", bytes.NewReader([]byte(databytes)))
testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL)
resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes)))
if err != nil {
t.Fatal(err)
}
@ -140,33 +143,27 @@ func TestBzzResourceMultihash(t *testing.T) {
log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
// our mutable resource "name"
keybytes := "foo.eth"
topic, _ := mru.NewTopic("foo.eth", nil)
updateRequest := mru.NewFirstRequest(topic)
updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: keybytes,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
if err != nil {
t.Fatal(err)
}
updateRequest.SetData(mh, true)
updateRequest.SetData(mh)
if err := updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
body, err := updateRequest.MarshalJSON()
testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
query := testUrl.Query()
body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted
query.Set("manifest", "1") // indicate we want a manifest back
testUrl.RawQuery = query.Encode()
// create the multihash update
url = fmt.Sprintf("%s/bzz-resource:/", srv.URL)
resp, err = http.Post(url, "application/json", bytes.NewReader(body))
resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@ -184,14 +181,14 @@ func TestBzzResourceMultihash(t *testing.T) {
t.Fatalf("data %s could not be unmarshaled: %v", b, err)
}
correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
if rsrcResp.Hex() != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
}
// get bzz manifest transparent resource resolve
url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(url)
testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testBzzUrl)
if err != nil {
t.Fatal(err)
}
@ -215,39 +212,38 @@ func TestBzzResource(t *testing.T) {
defer srv.Close()
// our mutable resource "name"
keybytes := "foo.eth"
// data of update 1
databytes := make([]byte, 666)
_, err := rand.Read(databytes)
update1Data := make([]byte, 666)
update1Timestamp := srv.CurrentTime
_, err := rand.Read(update1Data)
if err != nil {
t.Fatal(err)
}
//data for update 2
update2Data := []byte("foo")
updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: keybytes,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
topic, _ := mru.NewTopic("foo.eth", nil)
updateRequest := mru.NewFirstRequest(topic)
if err != nil {
t.Fatal(err)
}
updateRequest.SetData(databytes, false)
updateRequest.SetData(update1Data)
if err := updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
body, err := updateRequest.MarshalJSON()
// creates resource and sets update 1
testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
urlQuery := testUrl.Query()
body := updateRequest.AppendValues(urlQuery) // this adds all query parameters
urlQuery.Set("manifest", "1") // indicate we want a manifest back
testUrl.RawQuery = urlQuery.Encode()
// creates resource and sets update 1
url := fmt.Sprintf("%s/bzz-resource:/", srv.URL)
resp, err := http.Post(url, "application/json", bytes.NewReader(body))
resp, err := http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@ -265,14 +261,14 @@ func TestBzzResource(t *testing.T) {
t.Fatalf("data %s could not be unmarshaled: %v", b, err)
}
correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
if rsrcResp.Hex() != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
}
// get the manifest
url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp)
resp, err = http.Get(url)
testRawUrl := fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testRawUrl)
if err != nil {
t.Fatal(err)
}
@ -292,20 +288,20 @@ func TestBzzResource(t *testing.T) {
if len(manifest.Entries) != 1 {
t.Fatalf("Manifest has %d entries", len(manifest.Entries))
}
correctRootKeyHex := "68f7ba07ac8867a4c841a4d4320e3cdc549df23702dc7285fcb6acf65df48562"
if manifest.Entries[0].Hash != correctRootKeyHex {
t.Fatalf("Expected manifest path '%s', got '%s'", correctRootKeyHex, manifest.Entries[0].Hash)
correctViewHex := "0x666f6f2e65746800000000000000000000000000000000000000000000000000c96aaa54e2d44c299564da76e1cd3184a2386b8d"
if manifest.Entries[0].ResourceView.Hex() != correctViewHex {
t.Fatalf("Expected manifest Resource View '%s', got '%s'", correctViewHex, manifest.Entries[0].ResourceView.Hex())
}
// get bzz manifest transparent resource resolve
url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(url)
testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testBzzUrl)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
if resp.StatusCode == http.StatusOK {
t.Fatal("Expected error status since resource is not multihash. Received 200 OK")
}
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
@ -313,8 +309,8 @@ func TestBzzResource(t *testing.T) {
}
// get non-existent name, should fail
url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
resp, err = http.Get(url)
testBzzResUrl := fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
resp, err = http.Get(testBzzResUrl)
if err != nil {
t.Fatal(err)
}
@ -327,8 +323,8 @@ func TestBzzResource(t *testing.T) {
// get latest update (1.1) through resource directly
log.Info("get update latest = 1.1", "addr", correctManifestAddrHex)
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(testBzzResUrl)
if err != nil {
t.Fatal(err)
}
@ -340,16 +336,18 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, b) {
t.Fatalf("Expected body '%x', got '%x'", databytes, b)
if !bytes.Equal(update1Data, b) {
t.Fatalf("Expected body '%x', got '%x'", update1Data, b)
}
// update 2
// Move the clock ahead 1 second
srv.CurrentTime++
log.Info("update 2")
// 1.- get metadata about this resource
url = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url + "meta")
testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
resp, err = http.Get(testBzzResUrl + "?meta=1")
if err != nil {
t.Fatal(err)
}
@ -365,17 +363,19 @@ func TestBzzResource(t *testing.T) {
if err = updateRequest.UnmarshalJSON(b); err != nil {
t.Fatalf("Error decoding resource metadata: %s", err)
}
data := []byte("foo")
updateRequest.SetData(data, false)
updateRequest.SetData(update2Data)
if err = updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
body, err = updateRequest.MarshalJSON()
testUrl, err = url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
urlQuery = testUrl.Query()
body = updateRequest.AppendValues(urlQuery) // this adds all query parameters
testUrl.RawQuery = urlQuery.Encode()
resp, err = http.Post(url, "application/json", bytes.NewReader(body))
resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@ -386,8 +386,8 @@ func TestBzzResource(t *testing.T) {
// get latest update (1.2) through resource directly
log.Info("get update 1.2")
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(testBzzResUrl)
if err != nil {
t.Fatal(err)
}
@ -399,33 +399,23 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, b) {
t.Fatalf("Expected body '%x', got '%x'", data, b)
if !bytes.Equal(update2Data, b) {
t.Fatalf("Expected body '%x', got '%x'", update2Data, b)
}
// get latest update (1.2) with specified period
log.Info("get update latest = 1.2")
url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
// test manifest-less queries
log.Info("get first update in update1Timestamp via direct query")
query := mru.NewQuery(&updateRequest.View, update1Timestamp, lookup.NoClue)
urlq, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
}
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, b) {
t.Fatalf("Expected body '%x', got '%x'", data, b)
}
// get first update (1.1) with specified period and version
log.Info("get first update 1.1")
url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
values := urlq.Query()
query.AppendValues(values) // this adds view query parameters
urlq.RawQuery = values.Encode()
resp, err = http.Get(urlq.String())
if err != nil {
t.Fatal(err)
}
@ -437,9 +427,10 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, b) {
t.Fatalf("Expected body '%x', got '%x'", databytes, b)
if !bytes.Equal(update1Data, b) {
t.Fatalf("Expected body '%x', got '%x'", update1Data, b)
}
}
func TestBzzGetPath(t *testing.T) {

25
swarm/api/manifest.go
View File

@ -27,6 +27,8 @@ import (
"strings"
"time"
"github.com/ethereum/go-ethereum/swarm/storage/mru"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage"
@ -46,14 +48,15 @@ type Manifest struct {
// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
Access *AccessEntry `json:"access,omitempty"`
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
Access *AccessEntry `json:"access,omitempty"`
ResourceView *mru.View `json:"resourceView,omitempty"`
}
// ManifestList represents the result of listing files in a manifest
@ -79,11 +82,11 @@ func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address,
// Manifest hack for supporting Mutable Resource Updates from the bzz: scheme
// see swarm/api/api.go:API.Get() for more information
func (a *API) NewResourceManifest(ctx context.Context, resourceAddr string) (storage.Address, error) {
func (a *API) NewResourceManifest(ctx context.Context, view *mru.View) (storage.Address, error) {
var manifest Manifest
entry := ManifestEntry{
Hash: resourceAddr,
ContentType: ResourceContentType,
ResourceView: view,
ContentType: ResourceContentType,
}
manifest.Entries = append(manifest.Entries, entry)
data, err := json.Marshal(&manifest)