Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency swarm/storage/mru/lookup: fixed getBaseTime Added NewEpoch constructor swarm/api/client: better error handling in GetResource() swarm/storage/mru: Renamed structures. Renamed ResourceMetadata to ResourceID. Renamed ResourceID.Name to ResourceID.Topic swarm/storage/mru: Added binarySerializer interface and test tools swarm/storage/mru/lookup: Changed base time to time and + marshallers swarm/storage/mru: Added ResourceID (former resourceMetadata) swarm/storage/mru: Added ResourceViewId and serialization tests swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals swarm/storage/mru: Fixes as per review comments cmd/swarm: reworded resource create/update help text regarding topic swarm/storage/mru: Added UpdateLookup and serializer tests swarm/storage/mru: Added UpdateHeader, serializers and tests swarm/storage/mru: changed UpdateAddr / epoch to Base() swarm/storage/mru: Added resourceUpdate serializer and tests swarm/storage/mru: Added SignedResourceUpdate tests and serializers swarm/storage/mru/lookup: fixed GetFirstEpoch bug swarm/storage/mru: refactor, comments, cleanup Also added tests for Topic swarm/storage/mru: handler tests pass swarm/storage/mru: all resource package tests pass swarm/storage/mru: resource test pass after adding timestamp checking support swarm/storage/mru: Added JSON serializers to ResourceIDView structures swarm/storage/mru: Sever, client, API test pass swarm/storage/mru: server test pass swarm/storage/mru: Added topic length check swarm/storage/mru: removed some literals, improved "previous lookup" test case swarm/storage/mru: some fixes and comments as per review swarm/storage/mru: first working version without metadata chunk swarm/storage/mru: Various fixes as per review swarm/storage/mru: client test pass swarm/storage/mru: resource query strings and manifest-less queries swarm/storage/mru: simplify naming swarm/storage/mru: first autofreq working version swarm/storage/mru: renamed ToValues to AppendValues swarm/resource/mru: Added ToValues / FromValues for URL query strings swarm/storage/mru: Changed POST resource to work with query strings. No more JSON. swarm/storage/mru: removed resourceid swarm/storage/mru: Opened up structures swarm/storage/mru: Merged Request and SignedResourceUpdate swarm/storage/mru: removed initial data from CLI resource create swarm/storage/mru: Refactor Topic as a direct fixed-length array swarm/storage/mru/lookup: Comprehensive GetNextLevel tests swarm/storage/mru: Added comments Added length checks in Topic swarm/storage/mru: fixes in tests and some code comments swarm/storage/mru/lookup: new optimized lookup algorithm swarm/api: moved getResourceView to api out of server swarm/storage/mru: Lookup algorithm working swarm/storage/mru: comments and renamed NewLookupParams Deleted commented code swarm/storage/mru/lookup: renamed Epoch.LaterThan to After swarm/storage/mru/lookup: Comments and tidying naming swarm/storage/mru: fix lookup algorithm swarm/storage/mru: exposed lookup hint removed updateheader swarm/storage/mru/lookup: changed GetNextEpoch for initial values swarm/storage/mru: resource tests pass swarm/storage/mru: valueSerializer interface and tests swarm/storage/mru/lookup: Comments, improvements, fixes, more tests swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query swarm/storage/mru: renamed query receiver var swarm/cmd: MRU CLI tests * cmd/swarm: remove rogue fmt * swarm/storage/mru: Add version / header for future use * swarm/storage/mru: Fixes/comments as per review cmd/swarm: remove rogue fmt swarm/storage/mru: Add version / header for future use- * swarm/storage/mru: fix linter errors * cmd/swarm: Speeded up TestCLIResourceUpdate
This commit is contained in:
Javier Peletier
committed by
Martin Holst Swende
Martin Holst Swende
parent
0da3b17a11
commit
2c110c81ee
176
swarm/api/api.go
176
swarm/api/api.go
@@ -29,6 +29,8 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
|
||||
|
||||
"bytes"
|
||||
"mime"
|
||||
"path/filepath"
|
||||
@@ -401,77 +403,54 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
|
||||
|
||||
// we need to do some extra work if this is a mutable resource manifest
|
||||
if entry.ContentType == ResourceContentType {
|
||||
|
||||
// get the resource rootAddr
|
||||
log.Trace("resource type", "menifestAddr", manifestAddr, "hash", entry.Hash)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
rootAddr := storage.Address(common.FromHex(entry.Hash))
|
||||
rsrc, err := a.resource.Load(ctx, rootAddr)
|
||||
if entry.ResourceView == nil {
|
||||
return reader, mimeType, status, nil, fmt.Errorf("Cannot decode ResourceView in manifest")
|
||||
}
|
||||
_, err := a.resource.Lookup(ctx, mru.NewQueryLatest(entry.ResourceView, lookup.NoClue))
|
||||
if err != nil {
|
||||
apiGetNotFound.Inc(1)
|
||||
status = http.StatusNotFound
|
||||
log.Debug(fmt.Sprintf("get resource content error: %v", err))
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
|
||||
// use this key to retrieve the latest update
|
||||
params := mru.LookupLatest(rootAddr)
|
||||
rsrc, err = a.resource.Lookup(ctx, params)
|
||||
// get the data of the update
|
||||
_, rsrcData, err := a.resource.GetContent(entry.ResourceView)
|
||||
if err != nil {
|
||||
apiGetNotFound.Inc(1)
|
||||
status = http.StatusNotFound
|
||||
log.Debug(fmt.Sprintf("get resource content error: %v", err))
|
||||
log.Warn(fmt.Sprintf("get resource content error: %v", err))
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
|
||||
// if it's multihash, we will transparently serve the content this multihash points to
|
||||
// \TODO this resolve is rather expensive all in all, review to see if it can be achieved cheaper
|
||||
if rsrc.Multihash() {
|
||||
// extract multihash
|
||||
decodedMultihash, err := multihash.FromMultihash(rsrcData)
|
||||
if err != nil {
|
||||
apiGetInvalid.Inc(1)
|
||||
status = http.StatusUnprocessableEntity
|
||||
log.Warn("invalid resource multihash", "err", err)
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
manifestAddr = storage.Address(decodedMultihash)
|
||||
log.Trace("resource is multihash", "key", manifestAddr)
|
||||
|
||||
// get the data of the update
|
||||
_, rsrcData, err := a.resource.GetContent(rootAddr)
|
||||
if err != nil {
|
||||
apiGetNotFound.Inc(1)
|
||||
status = http.StatusNotFound
|
||||
log.Warn(fmt.Sprintf("get resource content error: %v", err))
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
// get the manifest the multihash digest points to
|
||||
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
|
||||
if err != nil {
|
||||
apiGetNotFound.Inc(1)
|
||||
status = http.StatusNotFound
|
||||
log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
|
||||
// validate that data as multihash
|
||||
decodedMultihash, err := multihash.FromMultihash(rsrcData)
|
||||
if err != nil {
|
||||
apiGetInvalid.Inc(1)
|
||||
status = http.StatusUnprocessableEntity
|
||||
log.Warn("invalid resource multihash", "err", err)
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
manifestAddr = storage.Address(decodedMultihash)
|
||||
log.Trace("resource is multihash", "key", manifestAddr)
|
||||
|
||||
// get the manifest the multihash digest points to
|
||||
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt)
|
||||
if err != nil {
|
||||
apiGetNotFound.Inc(1)
|
||||
status = http.StatusNotFound
|
||||
log.Warn(fmt.Sprintf("loadManifestTrie (resource multihash) error: %v", err))
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
|
||||
// finally, get the manifest entry
|
||||
// it will always be the entry on path ""
|
||||
entry, _ = trie.getEntry(path)
|
||||
if entry == nil {
|
||||
status = http.StatusNotFound
|
||||
apiGetNotFound.Inc(1)
|
||||
err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
|
||||
log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path)
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
|
||||
} else {
|
||||
// data is returned verbatim since it's not a multihash
|
||||
return rsrc, "application/octet-stream", http.StatusOK, nil, nil
|
||||
// finally, get the manifest entry
|
||||
// it will always be the entry on path ""
|
||||
entry, _ = trie.getEntry(path)
|
||||
if entry == nil {
|
||||
status = http.StatusNotFound
|
||||
apiGetNotFound.Inc(1)
|
||||
err = fmt.Errorf("manifest (resource multihash) entry for '%s' not found", path)
|
||||
log.Trace("manifest (resource multihash) entry not found", "key", manifestAddr, "path", path)
|
||||
return reader, mimeType, status, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -966,37 +945,27 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver
|
||||
}
|
||||
|
||||
// ResourceLookup finds mutable resource updates at specific periods and versions
|
||||
func (a *API) ResourceLookup(ctx context.Context, params *mru.LookupParams) (string, []byte, error) {
|
||||
var err error
|
||||
rsrc, err := a.resource.Load(ctx, params.RootAddr())
|
||||
func (a *API) ResourceLookup(ctx context.Context, query *mru.Query) ([]byte, error) {
|
||||
_, err := a.resource.Lookup(ctx, query)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
_, err = a.resource.Lookup(ctx, params)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
return nil, err
|
||||
}
|
||||
var data []byte
|
||||
_, data, err = a.resource.GetContent(params.RootAddr())
|
||||
_, data, err = a.resource.GetContent(&query.View)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
return nil, err
|
||||
}
|
||||
return rsrc.Name(), data, nil
|
||||
}
|
||||
|
||||
// Create Mutable resource
|
||||
func (a *API) ResourceCreate(ctx context.Context, request *mru.Request) error {
|
||||
return a.resource.New(ctx, request)
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ResourceNewRequest creates a Request object to update a specific mutable resource
|
||||
func (a *API) ResourceNewRequest(ctx context.Context, rootAddr storage.Address) (*mru.Request, error) {
|
||||
return a.resource.NewUpdateRequest(ctx, rootAddr)
|
||||
func (a *API) ResourceNewRequest(ctx context.Context, view *mru.View) (*mru.Request, error) {
|
||||
return a.resource.NewRequest(ctx, view)
|
||||
}
|
||||
|
||||
// ResourceUpdate updates a Mutable Resource with arbitrary data.
|
||||
// Upon retrieval the update will be retrieved verbatim as bytes.
|
||||
func (a *API) ResourceUpdate(ctx context.Context, request *mru.SignedResourceUpdate) (storage.Address, error) {
|
||||
func (a *API) ResourceUpdate(ctx context.Context, request *mru.Request) (storage.Address, error) {
|
||||
return a.resource.Update(ctx, request)
|
||||
}
|
||||
|
||||
@@ -1005,17 +974,62 @@ func (a *API) ResourceHashSize() int {
|
||||
return a.resource.HashSize
|
||||
}
|
||||
|
||||
// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk.
|
||||
func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) {
|
||||
// ErrCannotLoadResourceManifest is returned when looking up a resource manifest fails
|
||||
var ErrCannotLoadResourceManifest = errors.New("Cannot load resource manifest")
|
||||
|
||||
// ErrNotAResourceManifest is returned when the address provided returned something other than a valid manifest
|
||||
var ErrNotAResourceManifest = errors.New("Not a resource manifest")
|
||||
|
||||
// ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the Resource's view ID.
|
||||
func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (*mru.View, error) {
|
||||
trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load resource manifest: %v", err)
|
||||
return nil, ErrCannotLoadResourceManifest
|
||||
}
|
||||
|
||||
entry, _ := trie.getEntry("")
|
||||
if entry.ContentType != ResourceContentType {
|
||||
return nil, fmt.Errorf("not a resource manifest: %s", addr)
|
||||
return nil, ErrNotAResourceManifest
|
||||
}
|
||||
|
||||
return storage.Address(common.FromHex(entry.Hash)), nil
|
||||
return entry.ResourceView, nil
|
||||
}
|
||||
|
||||
// ErrCannotResolveResourceURI is returned when the ENS resolver is not able to translate a name to a resource
|
||||
var ErrCannotResolveResourceURI = errors.New("Cannot resolve Resource URI")
|
||||
|
||||
// ErrCannotResolveResourceView is returned when values provided are not enough or invalid to recreate a
|
||||
// resource view out of them.
|
||||
var ErrCannotResolveResourceView = errors.New("Cannot resolve resource view")
|
||||
|
||||
// ResolveResourceView attempts to extract View information out of the manifest, if provided
|
||||
// If not, it attempts to extract the View out of a set of key-value pairs
|
||||
func (a *API) ResolveResourceView(ctx context.Context, uri *URI, values mru.Values) (*mru.View, error) {
|
||||
var view *mru.View
|
||||
var err error
|
||||
if uri.Addr != "" {
|
||||
// resolve the content key.
|
||||
manifestAddr := uri.Address()
|
||||
if manifestAddr == nil {
|
||||
manifestAddr, err = a.Resolve(ctx, uri.Addr)
|
||||
if err != nil {
|
||||
return nil, ErrCannotResolveResourceURI
|
||||
}
|
||||
}
|
||||
|
||||
// get the resource view from the manifest
|
||||
view, err = a.ResolveResourceManifest(ctx, manifestAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug("handle.get.resource: resolved", "manifestkey", manifestAddr, "view", view.Hex())
|
||||
} else {
|
||||
var v mru.View
|
||||
if err := v.FromValues(values); err != nil {
|
||||
return nil, ErrCannotResolveResourceView
|
||||
|
||||
}
|
||||
view = &v
|
||||
}
|
||||
return view, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user