gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

Browse Source 
* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate
This commit is contained in:
Javier Peletier
2018-09-28 12:07:17 +02:00
committed by Martin Holst Swende
parent 0da3b17a11
commit 2c110c81ee
44 changed files with 3337 additions and 3022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
swarm/api/http/server_test.go
View File

@@ -30,12 +30,15 @@ import (
"math/big"
"mime/multipart"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@@ -121,8 +124,8 @@ func TestBzzResourceMultihash(t *testing.T) {
// add the data our multihash aliased manifest will point to
databytes := "bar"
url := fmt.Sprintf("%s/bzz:/", srv.URL)
resp, err := http.Post(url, "text/plain", bytes.NewReader([]byte(databytes)))
testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL)
resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes)))
if err != nil {
t.Fatal(err)
}
@@ -140,33 +143,27 @@ func TestBzzResourceMultihash(t *testing.T) {
log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
// our mutable resource "name"
keybytes := "foo.eth"
topic, _ := mru.NewTopic("foo.eth", nil)
updateRequest := mru.NewFirstRequest(topic)
updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: keybytes,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
if err != nil {
t.Fatal(err)
}
updateRequest.SetData(mh, true)
updateRequest.SetData(mh)
if err := updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
body, err := updateRequest.MarshalJSON()
testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
query := testUrl.Query()
body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted
query.Set("manifest", "1") // indicate we want a manifest back
testUrl.RawQuery = query.Encode()
// create the multihash update
url = fmt.Sprintf("%s/bzz-resource:/", srv.URL)
resp, err = http.Post(url, "application/json", bytes.NewReader(body))
resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@@ -184,14 +181,14 @@ func TestBzzResourceMultihash(t *testing.T) {
t.Fatalf("data %s could not be unmarshaled: %v", b, err)
}
correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
if rsrcResp.Hex() != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
}
// get bzz manifest transparent resource resolve
url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(url)
testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testBzzUrl)
if err != nil {
t.Fatal(err)
}
@@ -215,39 +212,38 @@ func TestBzzResource(t *testing.T) {
defer srv.Close()
// our mutable resource "name"
keybytes := "foo.eth"
// data of update 1
databytes := make([]byte, 666)
_, err := rand.Read(databytes)
update1Data := make([]byte, 666)
update1Timestamp := srv.CurrentTime
_, err := rand.Read(update1Data)
if err != nil {
t.Fatal(err)
}
//data for update 2
update2Data := []byte("foo")
updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
Name: keybytes,
Frequency: 13,
StartTime: srv.GetCurrentTime(),
Owner: signer.Address(),
})
topic, _ := mru.NewTopic("foo.eth", nil)
updateRequest := mru.NewFirstRequest(topic)
if err != nil {
t.Fatal(err)
}
updateRequest.SetData(databytes, false)
updateRequest.SetData(update1Data)
if err := updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
body, err := updateRequest.MarshalJSON()
// creates resource and sets update 1
testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
urlQuery := testUrl.Query()
body := updateRequest.AppendValues(urlQuery) // this adds all query parameters
urlQuery.Set("manifest", "1") // indicate we want a manifest back
testUrl.RawQuery = urlQuery.Encode()
// creates resource and sets update 1
url := fmt.Sprintf("%s/bzz-resource:/", srv.URL)
resp, err := http.Post(url, "application/json", bytes.NewReader(body))
resp, err := http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@@ -265,14 +261,14 @@ func TestBzzResource(t *testing.T) {
t.Fatalf("data %s could not be unmarshaled: %v", b, err)
}
correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
if rsrcResp.Hex() != correctManifestAddrHex {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
}
// get the manifest
url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp)
resp, err = http.Get(url)
testRawUrl := fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testRawUrl)
if err != nil {
t.Fatal(err)
}
@@ -292,20 +288,20 @@ func TestBzzResource(t *testing.T) {
if len(manifest.Entries) != 1 {
t.Fatalf("Manifest has %d entries", len(manifest.Entries))
}
correctRootKeyHex := "68f7ba07ac8867a4c841a4d4320e3cdc549df23702dc7285fcb6acf65df48562"
if manifest.Entries[0].Hash != correctRootKeyHex {
t.Fatalf("Expected manifest path '%s', got '%s'", correctRootKeyHex, manifest.Entries[0].Hash)
correctViewHex := "0x666f6f2e65746800000000000000000000000000000000000000000000000000c96aaa54e2d44c299564da76e1cd3184a2386b8d"
if manifest.Entries[0].ResourceView.Hex() != correctViewHex {
t.Fatalf("Expected manifest Resource View '%s', got '%s'", correctViewHex, manifest.Entries[0].ResourceView.Hex())
}
// get bzz manifest transparent resource resolve
url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(url)
testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
resp, err = http.Get(testBzzUrl)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
if resp.StatusCode == http.StatusOK {
t.Fatal("Expected error status since resource is not multihash. Received 200 OK")
}
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
@@ -313,8 +309,8 @@ func TestBzzResource(t *testing.T) {
}
// get non-existent name, should fail
url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
resp, err = http.Get(url)
testBzzResUrl := fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
resp, err = http.Get(testBzzResUrl)
if err != nil {
t.Fatal(err)
}
@@ -327,8 +323,8 @@ func TestBzzResource(t *testing.T) {
// get latest update (1.1) through resource directly
log.Info("get update latest = 1.1", "addr", correctManifestAddrHex)
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(testBzzResUrl)
if err != nil {
t.Fatal(err)
}
@@ -340,16 +336,18 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, b) {
t.Fatalf("Expected body '%x', got '%x'", databytes, b)
if !bytes.Equal(update1Data, b) {
t.Fatalf("Expected body '%x', got '%x'", update1Data, b)
}
// update 2
// Move the clock ahead 1 second
srv.CurrentTime++
log.Info("update 2")
// 1.- get metadata about this resource
url = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url + "meta")
testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
resp, err = http.Get(testBzzResUrl + "?meta=1")
if err != nil {
t.Fatal(err)
}
@@ -365,17 +363,19 @@ func TestBzzResource(t *testing.T) {
if err = updateRequest.UnmarshalJSON(b); err != nil {
t.Fatalf("Error decoding resource metadata: %s", err)
}
data := []byte("foo")
updateRequest.SetData(data, false)
updateRequest.SetData(update2Data)
if err = updateRequest.Sign(signer); err != nil {
t.Fatal(err)
}
body, err = updateRequest.MarshalJSON()
testUrl, err = url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
urlQuery = testUrl.Query()
body = updateRequest.AppendValues(urlQuery) // this adds all query parameters
testUrl.RawQuery = urlQuery.Encode()
resp, err = http.Post(url, "application/json", bytes.NewReader(body))
resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
@@ -386,8 +386,8 @@ func TestBzzResource(t *testing.T) {
// get latest update (1.2) through resource directly
log.Info("get update 1.2")
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
resp, err = http.Get(testBzzResUrl)
if err != nil {
t.Fatal(err)
}
@@ -399,33 +399,23 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, b) {
t.Fatalf("Expected body '%x', got '%x'", data, b)
if !bytes.Equal(update2Data, b) {
t.Fatalf("Expected body '%x', got '%x'", update2Data, b)
}
// get latest update (1.2) with specified period
log.Info("get update latest = 1.2")
url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
// test manifest-less queries
log.Info("get first update in update1Timestamp via direct query")
query := mru.NewQuery(&updateRequest.View, update1Timestamp, lookup.NoClue)
urlq, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("err %s", resp.Status)
}
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, b) {
t.Fatalf("Expected body '%x', got '%x'", data, b)
}
// get first update (1.1) with specified period and version
log.Info("get first update 1.1")
url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, correctManifestAddrHex)
resp, err = http.Get(url)
values := urlq.Query()
query.AppendValues(values) // this adds view query parameters
urlq.RawQuery = values.Encode()
resp, err = http.Get(urlq.String())
if err != nil {
t.Fatal(err)
}
@@ -437,9 +427,10 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, b) {
t.Fatalf("Expected body '%x', got '%x'", databytes, b)
if !bytes.Equal(update1Data, b) {
t.Fatalf("Expected body '%x', got '%x'", update1Data, b)
}
}
func TestBzzGetPath(t *testing.T) {