gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

Browse Source 
* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate
This commit is contained in:
Javier Peletier
2018-09-28 12:07:17 +02:00
committed by Martin Holst Swende
parent 0da3b17a11
commit 2c110c81ee
44 changed files with 3337 additions and 3022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
cmd/swarm/main.go
View File

@ -203,21 +203,29 @@ var (
Usage: "Number of recent chunks cached in memory (default 5000)",
EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
}
SwarmResourceMultihashFlag = cli.BoolFlag{
Name: "multihash",
Usage: "Determines how to interpret data for a resource update. If not present, data will be interpreted as raw, literal data that will be included in the resource",
SwarmCompressedFlag = cli.BoolFlag{
Name: "compressed",
Usage: "Prints encryption keys in compressed form",
}
SwarmResourceNameFlag = cli.StringFlag{
Name: "name",
Usage: "User-defined name for the new resource",
Usage: "User-defined name for the new resource, limited to 32 characters. If combined with topic, the resource will be a subtopic with this name",
}
SwarmResourceTopicFlag = cli.StringFlag{
Name: "topic",
Usage: "User-defined topic this resource is tracking, hex encoded. Limited to 64 hexadecimal characters",
}
SwarmResourceDataOnCreateFlag = cli.StringFlag{
Name: "data",
Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x",
}
SwarmCompressedFlag = cli.BoolFlag{
Name: "compressed",
Usage: "Prints encryption keys in compressed form",
SwarmResourceManifestFlag = cli.StringFlag{
Name: "manifest",
Usage: "Refers to the resource through a manifest",
}
SwarmResourceUserFlag = cli.StringFlag{
Name: "user",
Usage: "Indicates the user who updates the resource",
}
)
@ -347,27 +355,53 @@ func init() {
Action: resourceCreate,
CustomHelpTemplate: helpTemplate,
Name: "create",
Usage: "creates a new Mutable Resource",
ArgsUsage: "<frequency>",
Description: "creates a new Mutable Resource",
Flags: []cli.Flag{SwarmResourceNameFlag, SwarmResourceDataOnCreateFlag, SwarmResourceMultihashFlag},
Usage: "creates and publishes a new Mutable Resource manifest",
Description: `creates and publishes a new Mutable Resource manifest pointing to a specified user's updates about a particular topic.
The resource topic can be built in the following ways:
* use --topic to set the topic to an arbitrary binary hex string.
* use --name to set the topic to a human-readable name.
For example --name could be set to "profile-picture", meaning this Mutable Resource allows to get this user's current profile picture.
* use both --topic and --name to create named subtopics.
For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
the Mutable Resource tracks a discussion about that contract.
The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
it will then default to your local account (--bzzaccount)`,
Flags: []cli.Flag{SwarmResourceNameFlag, SwarmResourceTopicFlag, SwarmResourceUserFlag},
},
{
Action: resourceUpdate,
CustomHelpTemplate: helpTemplate,
Name: "update",
Usage: "updates the content of an existing Mutable Resource",
ArgsUsage: "<Manifest Address or ENS domain> <0x Hex data>",
Description: "updates the content of an existing Mutable Resource",
Flags: []cli.Flag{SwarmResourceMultihashFlag},
ArgsUsage: "<0x Hex data>",
Description: `publishes a new update on the specified topic
The resource topic can be built in the following ways:
* use --topic to set the topic to an arbitrary binary hex string.
* use --name to set the topic to a human-readable name.
For example --name could be set to "profile-picture", meaning this Mutable Resource allows to get this user's current profile picture.
* use both --topic and --name to create named subtopics.
For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
the Mutable Resource tracks a discussion about that contract.
If you have a manifest, you can specify it with --manifest to refer to the resource,
instead of using --topic / --name
`,
Flags: []cli.Flag{SwarmResourceManifestFlag, SwarmResourceNameFlag, SwarmResourceTopicFlag},
},
{
Action: resourceInfo,
CustomHelpTemplate: helpTemplate,
Name: "info",
Usage: "obtains information about an existing Mutable Resource",
ArgsUsage: "<Manifest Address or ENS domain>",
Description: "obtains information about an existing Mutable Resource",
Description: `obtains information about an existing Mutable Resource
The topic can be specified directly with the --topic flag as an hex string
If no topic is specified, the default topic (zero) will be used
The --name flag can be used to specify subtopics with a specific name.
The --user flag allows to refer to a user other than yourself. If not specified,
it will then default to your local account (--bzzaccount)
If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
to refer to the resource`,
Flags: []cli.Flag{SwarmResourceManifestFlag, SwarmResourceNameFlag, SwarmResourceTopicFlag, SwarmResourceUserFlag},
},
},
},

135
cmd/swarm/mru.go
View File

@ -19,10 +19,11 @@ package main
import (
"fmt"
"strconv"
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
@ -34,62 +35,38 @@ func NewGenericSigner(ctx *cli.Context) mru.Signer {
return mru.NewGenericSigner(getPrivKey(ctx))
}
func getTopic(ctx *cli.Context) (topic mru.Topic) {
var name = ctx.String(SwarmResourceNameFlag.Name)
var relatedTopic = ctx.String(SwarmResourceTopicFlag.Name)
var relatedTopicBytes []byte
var err error
if relatedTopic != "" {
relatedTopicBytes, err = hexutil.Decode(relatedTopic)
if err != nil {
utils.Fatalf("Error parsing topic: %s", err)
}
}
topic, err = mru.NewTopic(name, relatedTopicBytes)
if err != nil {
utils.Fatalf("Error parsing topic: %s", err)
}
return topic
}
// swarm resource create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
// swarm resource update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
// swarm resource info <Manifest Address or ENS domain>
func resourceCreate(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
initialData = ctx.String(SwarmResourceDataOnCreateFlag.Name)
name = ctx.String(SwarmResourceNameFlag.Name)
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
)
if len(args) < 1 {
fmt.Println("Incorrect number of arguments")
cli.ShowCommandHelpAndExit(ctx, "create", 1)
return
}
signer := NewGenericSigner(ctx)
frequency, err := strconv.ParseUint(args[0], 10, 64)
if err != nil {
fmt.Printf("Frequency formatting error: %s\n", err.Error())
cli.ShowCommandHelpAndExit(ctx, "create", 1)
return
}
metadata := mru.ResourceMetadata{
Name: name,
Frequency: frequency,
Owner: signer.Address(),
}
var newResourceRequest *mru.Request
if initialData != "" {
initialDataBytes, err := hexutil.Decode(initialData)
if err != nil {
fmt.Printf("Error parsing data: %s\n", err.Error())
cli.ShowCommandHelpAndExit(ctx, "create", 1)
return
}
newResourceRequest, err = mru.NewCreateUpdateRequest(&metadata)
if err != nil {
utils.Fatalf("Error creating new resource request: %s", err)
}
newResourceRequest.SetData(initialDataBytes, multihash)
if err = newResourceRequest.Sign(signer); err != nil {
utils.Fatalf("Error signing resource update: %s", err.Error())
}
} else {
newResourceRequest, err = mru.NewCreateRequest(&metadata)
if err != nil {
utils.Fatalf("Error creating new resource request: %s", err)
}
}
newResourceRequest := mru.NewFirstRequest(getTopic(ctx))
newResourceRequest.View.User = resourceGetUser(ctx)
manifestAddress, err := client.CreateResource(newResourceRequest)
if err != nil {
@ -104,32 +81,43 @@ func resourceUpdate(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
manifestAddressOrDomain = ctx.String(SwarmResourceManifestFlag.Name)
)
if len(args) < 2 {
if len(args) < 1 {
fmt.Println("Incorrect number of arguments")
cli.ShowCommandHelpAndExit(ctx, "update", 1)
return
}
signer := NewGenericSigner(ctx)
manifestAddressOrDomain := args[0]
data, err := hexutil.Decode(args[1])
data, err := hexutil.Decode(args[0])
if err != nil {
utils.Fatalf("Error parsing data: %s", err.Error())
return
}
var updateRequest *mru.Request
var query *mru.Query
if manifestAddressOrDomain == "" {
query = new(mru.Query)
query.User = signer.Address()
query.Topic = getTopic(ctx)
}
// Retrieve resource status and metadata out of the manifest
updateRequest, err := client.GetResourceMetadata(manifestAddressOrDomain)
updateRequest, err = client.GetResourceMetadata(query, manifestAddressOrDomain)
if err != nil {
utils.Fatalf("Error retrieving resource status: %s", err.Error())
}
// set the new data
updateRequest.SetData(data, multihash)
updateRequest.SetData(data)
// sign update
if err = updateRequest.Sign(signer); err != nil {
@ -146,17 +134,19 @@ func resourceUpdate(ctx *cli.Context) {
func resourceInfo(ctx *cli.Context) {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
manifestAddressOrDomain = ctx.String(SwarmResourceManifestFlag.Name)
)
args := ctx.Args()
if len(args) < 1 {
fmt.Println("Incorrect number of arguments.")
cli.ShowCommandHelpAndExit(ctx, "info", 1)
return
var query *mru.Query
if manifestAddressOrDomain == "" {
query = new(mru.Query)
query.Topic = getTopic(ctx)
query.User = resourceGetUser(ctx)
}
manifestAddressOrDomain := args[0]
metadata, err := client.GetResourceMetadata(manifestAddressOrDomain)
metadata, err := client.GetResourceMetadata(query, manifestAddressOrDomain)
if err != nil {
utils.Fatalf("Error retrieving resource metadata: %s", err.Error())
return
@ -167,3 +157,16 @@ func resourceInfo(ctx *cli.Context) {
}
fmt.Println(string(encodedMetadata))
}
func resourceGetUser(ctx *cli.Context) common.Address {
var user = ctx.String(SwarmResourceUserFlag.Name)
if user != "" {
return common.HexToAddress(user)
}
pk := getPrivKey(ctx)
if pk == nil {
utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount")
}
return crypto.PubkeyToAddress(pk.PublicKey)
}

182
cmd/swarm/mru_test.go Normal file
View File

@ -0,0 +1,182 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/storage/mru"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
)
func TestCLIResourceUpdate(t *testing.T) {
srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
return swarmhttp.NewServer(api, "")
}, nil)
log.Info("starting 1 node cluster")
defer srv.Close()
// create a private key file for signing
pkfile, err := ioutil.TempFile("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer pkfile.Close()
defer os.Remove(pkfile.Name())
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
privKey, _ := crypto.HexToECDSA(privkeyHex)
address := crypto.PubkeyToAddress(privKey.PublicKey)
// save the private key to a file
_, err = io.WriteString(pkfile, privkeyHex)
if err != nil {
t.Fatal(err)
}
// compose a topic. We'll be doing quotes about Miguel de Cervantes
var topic mru.Topic
subject := []byte("Miguel de Cervantes")
copy(topic[:], subject[:])
name := "quotes"
// prepare some data for the update
data := []byte("En boca cerrada no entran moscas")
hexData := hexutil.Encode(data)
flags := []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkfile.Name(),
"resource", "update",
"--topic", topic.Hex(),
"--name", name,
hexData}
// create an update and expect an exit without errors
log.Info(fmt.Sprintf("updating a resource with 'swarm resource update'"))
cmd := runSwarm(t, flags...)
cmd.ExpectExit()
// now try to get the update using the client
client := swarm.NewClient(srv.URL)
if err != nil {
t.Fatal(err)
}
// build the same topic as before, this time
// we use NewTopic to create a topic automatically.
topic, err = mru.NewTopic(name, subject)
if err != nil {
t.Fatal(err)
}
// View configures whose updates we will be looking up.
view := mru.View{
Topic: topic,
User: address,
}
// Build a query to get the latest update
query := mru.NewQueryLatest(&view, lookup.NoClue)
// retrieve content!
reader, err := client.GetResource(query, "")
if err != nil {
t.Fatal(err)
}
retrieved, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
// check we retrieved the sent information
if !bytes.Equal(data, retrieved) {
t.Fatalf("Received %s, expected %s", retrieved, data)
}
// Now retrieve info for the next update
flags = []string{
"--bzzapi", srv.URL,
"resource", "info",
"--topic", topic.Hex(),
"--user", address.Hex(),
}
log.Info(fmt.Sprintf("getting resource info with 'swarm resource info'"))
cmd = runSwarm(t, flags...)
_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
cmd.ExpectExit()
// verify we can deserialize the result as a valid JSON
var request mru.Request
err = json.Unmarshal([]byte(matches[0]), &request)
if err != nil {
t.Fatal(err)
}
// make sure the retrieved view is the same
if request.View != view {
t.Fatalf("Expected view to be: %s, got %s", view, request.View)
}
// test publishing a manifest
flags = []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkfile.Name(),
"resource", "create",
"--topic", topic.Hex(),
}
log.Info(fmt.Sprintf("Publishing manifest with 'swarm resource create'"))
cmd = runSwarm(t, flags...)
_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
cmd.ExpectExit()
manifestAddress := matches[0] // read the received resource manifest
// now attempt to lookup the latest update using a manifest instead
reader, err = client.GetResource(nil, manifestAddress)
if err != nil {
t.Fatal(err)
}
retrieved, err = ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, retrieved) {
t.Fatalf("Received %s, expected %s", retrieved, data)
}
}

BIN
cmd/swarm/swarm Executable file
View File

Binary file not shown.