swarm: codebase split from go-ethereum (#1405)
This commit is contained in:
Rafael Matias
committed by
Anton Evangelatov
Anton Evangelatov
parent
7a22da98b9
commit
b046760db1
291
cmd/swarm-smoke/feed_upload_and_sync.go
Normal file
291
cmd/swarm-smoke/feed_upload_and_sync.go
Normal file
@@ -0,0 +1,291 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethersphere/swarm/storage/feed"
|
||||
"github.com/ethersphere/swarm/testutil"
|
||||
"github.com/pborman/uuid"
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
const (
|
||||
feedRandomDataLength = 8
|
||||
)
|
||||
|
||||
func feedUploadAndSyncCmd(ctx *cli.Context) error {
|
||||
errc := make(chan error)
|
||||
|
||||
go func() {
|
||||
errc <- feedUploadAndSync(ctx)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-errc:
|
||||
if err != nil {
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
|
||||
}
|
||||
return err
|
||||
case <-time.After(time.Duration(timeout) * time.Second):
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
|
||||
|
||||
return fmt.Errorf("timeout after %v sec", timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func feedUploadAndSync(c *cli.Context) error {
|
||||
log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing")
|
||||
|
||||
// create a random private key to sign updates with and derive the address
|
||||
pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pkFile.Close()
|
||||
defer os.Remove(pkFile.Name())
|
||||
|
||||
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976"
|
||||
privKey, err := crypto.HexToECDSA(privkeyHex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
user := crypto.PubkeyToAddress(privKey.PublicKey)
|
||||
userHex := hexutil.Encode(user.Bytes())
|
||||
|
||||
// save the private key to a file
|
||||
_, err = io.WriteString(pkFile, privkeyHex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// keep hex strings for topic and subtopic
|
||||
var topicHex string
|
||||
var subTopicHex string
|
||||
|
||||
// and create combination hex topics for bzz-feed retrieval
|
||||
// xor'ed with topic (zero-value topic if no topic)
|
||||
var subTopicOnlyHex string
|
||||
var mergedSubTopicHex string
|
||||
|
||||
// generate random topic and subtopic and put a hex on them
|
||||
topicBytes, err := generateRandomData(feed.TopicLength)
|
||||
topicHex = hexutil.Encode(topicBytes)
|
||||
subTopicBytes, err := generateRandomData(8)
|
||||
subTopicHex = hexutil.Encode(subTopicBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:])
|
||||
subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:])
|
||||
|
||||
// create feed manifest, topic only
|
||||
var out bytes.Buffer
|
||||
cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex)
|
||||
cmd.Stdout = &out
|
||||
log.Debug("create feed manifest topic cmd", "cmd", cmd)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
|
||||
if len(manifestWithTopic) != 64 {
|
||||
return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic)
|
||||
}
|
||||
log.Debug("create topic feed", "manifest", manifestWithTopic)
|
||||
out.Reset()
|
||||
|
||||
// create feed manifest, subtopic only
|
||||
cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex)
|
||||
cmd.Stdout = &out
|
||||
log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
|
||||
if len(manifestWithSubTopic) != 64 {
|
||||
return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic)
|
||||
}
|
||||
log.Debug("create subtopic feed", "manifest", manifestWithTopic)
|
||||
out.Reset()
|
||||
|
||||
// create feed manifest, merged topic
|
||||
cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
|
||||
cmd.Stdout = &out
|
||||
log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
|
||||
if len(manifestWithMergedTopic) != 64 {
|
||||
return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic)
|
||||
}
|
||||
log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic)
|
||||
out.Reset()
|
||||
|
||||
// create test data
|
||||
data, err := generateRandomData(feedRandomDataLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h := md5.New()
|
||||
h.Write(data)
|
||||
dataHash := h.Sum(nil)
|
||||
dataHex := hexutil.Encode(data)
|
||||
|
||||
// update with topic
|
||||
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex)
|
||||
cmd.Stdout = &out
|
||||
log.Debug("update feed manifest topic cmd", "cmd", cmd)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("feed update topic", "out", out)
|
||||
out.Reset()
|
||||
|
||||
// update with subtopic
|
||||
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex)
|
||||
cmd.Stdout = &out
|
||||
log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("feed update subtopic", "out", out)
|
||||
out.Reset()
|
||||
|
||||
// update with merged topic
|
||||
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
|
||||
cmd.Stdout = &out
|
||||
log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("feed update mergedtopic", "out", out)
|
||||
out.Reset()
|
||||
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
// retrieve the data
|
||||
wg := sync.WaitGroup{}
|
||||
for _, host := range hosts {
|
||||
// raw retrieve, topic only
|
||||
for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
|
||||
wg.Add(1)
|
||||
ruid := uuid.New()[:8]
|
||||
go func(hex string, endpoint string, ruid string) {
|
||||
for {
|
||||
err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
}(hex, httpEndpoint(host), ruid)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
log.Info("all endpoints synced random data successfully")
|
||||
|
||||
// upload test file
|
||||
log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
|
||||
|
||||
randomBytes := testutil.RandomBytes(seed, filesize*1000)
|
||||
|
||||
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashBytes, err := hexutil.Decode("0x" + hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
multihashHex := hexutil.Encode(hashBytes)
|
||||
fileHash := h.Sum(nil)
|
||||
|
||||
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
|
||||
|
||||
// update file with topic
|
||||
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex)
|
||||
cmd.Stdout = &out
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("feed update topic", "out", out)
|
||||
out.Reset()
|
||||
|
||||
// update file with subtopic
|
||||
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex)
|
||||
cmd.Stdout = &out
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("feed update subtopic", "out", out)
|
||||
out.Reset()
|
||||
|
||||
// update file with merged topic
|
||||
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
|
||||
cmd.Stdout = &out
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("feed update mergedtopic", "out", out)
|
||||
out.Reset()
|
||||
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
for _, host := range hosts {
|
||||
|
||||
// manifest retrieve, topic only
|
||||
for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
|
||||
wg.Add(1)
|
||||
ruid := uuid.New()[:8]
|
||||
go func(url string, endpoint string, ruid string) {
|
||||
for {
|
||||
err := fetch(url, endpoint, fileHash, ruid)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
}(url, httpEndpoint(host), ruid)
|
||||
}
|
||||
|
||||
}
|
||||
wg.Wait()
|
||||
log.Info("all endpoints synced random file successfully")
|
||||
|
||||
return nil
|
||||
}
|
||||
195
cmd/swarm-smoke/main.go
Normal file
195
cmd/swarm-smoke/main.go
Normal file
@@ -0,0 +1,195 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
gethmetrics "github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethereum/go-ethereum/metrics/influxdb"
|
||||
swarmmetrics "github.com/ethersphere/swarm/metrics"
|
||||
"github.com/ethersphere/swarm/tracing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
|
||||
)
|
||||
|
||||
var (
|
||||
allhosts string
|
||||
hosts []string
|
||||
filesize int
|
||||
syncDelay bool
|
||||
inputSeed int
|
||||
httpPort int
|
||||
wsPort int
|
||||
verbosity int
|
||||
timeout int
|
||||
single bool
|
||||
onlyUpload bool
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
app := cli.NewApp()
|
||||
app.Name = "smoke-test"
|
||||
app.Usage = ""
|
||||
|
||||
app.Flags = []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "hosts",
|
||||
Value: "",
|
||||
Usage: "comma-separated list of swarm hosts",
|
||||
Destination: &allhosts,
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "http-port",
|
||||
Value: 80,
|
||||
Usage: "http port",
|
||||
Destination: &httpPort,
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "ws-port",
|
||||
Value: 8546,
|
||||
Usage: "ws port",
|
||||
Destination: &wsPort,
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "seed",
|
||||
Value: 0,
|
||||
Usage: "input seed in case we need deterministic upload",
|
||||
Destination: &inputSeed,
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "filesize",
|
||||
Value: 1024,
|
||||
Usage: "file size for generated random file in KB",
|
||||
Destination: &filesize,
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "sync-delay",
|
||||
Usage: "wait for content to be synced",
|
||||
Destination: &syncDelay,
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "verbosity",
|
||||
Value: 1,
|
||||
Usage: "verbosity",
|
||||
Destination: &verbosity,
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "timeout",
|
||||
Value: 180,
|
||||
Usage: "timeout in seconds after which kill the process",
|
||||
Destination: &timeout,
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "single",
|
||||
Usage: "whether to fetch content from a single node or from all nodes",
|
||||
Destination: &single,
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "only-upload",
|
||||
Usage: "whether to only upload content to a single node without fetching",
|
||||
Destination: &onlyUpload,
|
||||
},
|
||||
}
|
||||
|
||||
app.Flags = append(app.Flags, []cli.Flag{
|
||||
utils.MetricsEnabledFlag,
|
||||
swarmmetrics.MetricsInfluxDBEndpointFlag,
|
||||
swarmmetrics.MetricsInfluxDBDatabaseFlag,
|
||||
swarmmetrics.MetricsInfluxDBUsernameFlag,
|
||||
swarmmetrics.MetricsInfluxDBPasswordFlag,
|
||||
swarmmetrics.MetricsInfluxDBTagsFlag,
|
||||
}...)
|
||||
|
||||
app.Flags = append(app.Flags, tracing.Flags...)
|
||||
|
||||
app.Commands = []cli.Command{
|
||||
{
|
||||
Name: "upload_and_sync",
|
||||
Aliases: []string{"c"},
|
||||
Usage: "upload and sync",
|
||||
Action: wrapCliCommand("upload-and-sync", uploadAndSyncCmd),
|
||||
},
|
||||
{
|
||||
Name: "feed_sync",
|
||||
Aliases: []string{"f"},
|
||||
Usage: "feed update generate, upload and sync",
|
||||
Action: wrapCliCommand("feed-and-sync", feedUploadAndSyncCmd),
|
||||
},
|
||||
{
|
||||
Name: "upload_speed",
|
||||
Aliases: []string{"u"},
|
||||
Usage: "measure upload speed",
|
||||
Action: wrapCliCommand("upload-speed", uploadSpeedCmd),
|
||||
},
|
||||
{
|
||||
Name: "sliding_window",
|
||||
Aliases: []string{"s"},
|
||||
Usage: "measure network aggregate capacity",
|
||||
Action: wrapCliCommand("sliding-window", slidingWindowCmd),
|
||||
},
|
||||
}
|
||||
|
||||
sort.Sort(cli.FlagsByName(app.Flags))
|
||||
sort.Sort(cli.CommandsByName(app.Commands))
|
||||
|
||||
app.Before = func(ctx *cli.Context) error {
|
||||
tracing.Setup(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
app.After = func(ctx *cli.Context) error {
|
||||
return emitMetrics(ctx)
|
||||
}
|
||||
|
||||
err := app.Run(os.Args)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func emitMetrics(ctx *cli.Context) error {
|
||||
if gethmetrics.Enabled {
|
||||
var (
|
||||
endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name)
|
||||
database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name)
|
||||
username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name)
|
||||
password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name)
|
||||
tags = ctx.GlobalString(swarmmetrics.MetricsInfluxDBTagsFlag.Name)
|
||||
)
|
||||
|
||||
tagsMap := utils.SplitTagsFlag(tags)
|
||||
tagsMap["version"] = gitCommit
|
||||
tagsMap["filesize"] = fmt.Sprintf("%v", filesize)
|
||||
|
||||
return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", tagsMap)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
149
cmd/swarm-smoke/sliding_window.go
Normal file
149
cmd/swarm-smoke/sliding_window.go
Normal file
@@ -0,0 +1,149 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethersphere/swarm/testutil"
|
||||
"github.com/pborman/uuid"
|
||||
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
type uploadResult struct {
|
||||
hash string
|
||||
digest []byte
|
||||
}
|
||||
|
||||
func slidingWindowCmd(ctx *cli.Context) error {
|
||||
errc := make(chan error)
|
||||
|
||||
go func() {
|
||||
errc <- slidingWindow(ctx)
|
||||
}()
|
||||
|
||||
err := <-errc
|
||||
if err != nil {
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func slidingWindow(ctx *cli.Context) error {
|
||||
var hashes []uploadResult //swarm hashes of the uploads
|
||||
nodes := len(hosts)
|
||||
log.Info("sliding window test started", "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)
|
||||
uploadedBytes := 0
|
||||
networkDepth := 0
|
||||
errored := false
|
||||
|
||||
outer:
|
||||
for {
|
||||
seed = int(time.Now().UTC().UnixNano())
|
||||
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
|
||||
|
||||
t1 := time.Now()
|
||||
|
||||
randomBytes := testutil.RandomBytes(seed, filesize*1000)
|
||||
|
||||
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1)
|
||||
metrics.GetOrRegisterGauge("sliding-window.upload-depth", nil).Update(int64(len(hashes)))
|
||||
|
||||
fhash, err := digest(bytes.NewReader(randomBytes))
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "wait for sync", syncDelay)
|
||||
hashes = append(hashes, uploadResult{hash: hash, digest: fhash})
|
||||
|
||||
if syncDelay {
|
||||
waitToSync()
|
||||
}
|
||||
|
||||
uploadedBytes += filesize * 1000
|
||||
q := make(chan struct{}, 1)
|
||||
d := make(chan struct{})
|
||||
defer close(q)
|
||||
defer close(d)
|
||||
for i, v := range hashes {
|
||||
timeoutC := time.After(time.Duration(timeout) * time.Second)
|
||||
errored = false
|
||||
|
||||
task:
|
||||
for {
|
||||
select {
|
||||
case q <- struct{}{}:
|
||||
go func() {
|
||||
var start time.Time
|
||||
done := false
|
||||
for !done {
|
||||
log.Info("trying to retrieve hash", "hash", v.hash)
|
||||
idx := 1 + rand.Intn(len(hosts)-1)
|
||||
ruid := uuid.New()[:8]
|
||||
start = time.Now()
|
||||
// fetch hangs when swarm dies out, so we have to jump through a bit more hoops to actually
|
||||
// catch the timeout, but also allow this retry logic
|
||||
err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid)
|
||||
if err != nil {
|
||||
log.Error("error fetching hash", "err", err)
|
||||
continue
|
||||
}
|
||||
done = true
|
||||
}
|
||||
metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start)
|
||||
d <- struct{}{}
|
||||
}()
|
||||
case <-d:
|
||||
<-q
|
||||
break task
|
||||
case <-timeoutC:
|
||||
errored = true
|
||||
log.Error("error retrieving hash. timeout", "hash idx", i)
|
||||
metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
|
||||
break outer
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
networkDepth = i
|
||||
metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth))
|
||||
log.Info("sliding window test successfully fetched file", "currentDepth", networkDepth)
|
||||
// this test might take a long time to finish - but we'd like to see metrics while they accumulate and not just when
|
||||
// the test finishes. therefore emit the metrics on each iteration
|
||||
emitMetrics(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize)
|
||||
log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize)
|
||||
|
||||
return nil
|
||||
}
|
||||
376
cmd/swarm-smoke/upload_and_sync.go
Normal file
376
cmd/swarm-smoke/upload_and_sync.go
Normal file
@@ -0,0 +1,376 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
"github.com/ethersphere/swarm/chunk"
|
||||
"github.com/ethersphere/swarm/storage"
|
||||
"github.com/ethersphere/swarm/testutil"
|
||||
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
func uploadAndSyncCmd(ctx *cli.Context) error {
|
||||
// use input seed if it has been set
|
||||
if inputSeed != 0 {
|
||||
seed = inputSeed
|
||||
}
|
||||
|
||||
randomBytes := testutil.RandomBytes(seed, filesize*1000)
|
||||
|
||||
errc := make(chan error)
|
||||
|
||||
go func() {
|
||||
errc <- uploadAndSync(ctx, randomBytes)
|
||||
}()
|
||||
|
||||
var err error
|
||||
select {
|
||||
case err = <-errc:
|
||||
if err != nil {
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
|
||||
}
|
||||
case <-time.After(time.Duration(timeout) * time.Second):
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
|
||||
|
||||
err = fmt.Errorf("timeout after %v sec", timeout)
|
||||
}
|
||||
|
||||
// trigger debug functionality on randomBytes
|
||||
e := trackChunks(randomBytes[:], true)
|
||||
if e != nil {
|
||||
log.Error(e.Error())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func trackChunks(testData []byte, submitMetrics bool) error {
|
||||
addrs, err := getAllRefs(testData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, ref := range addrs {
|
||||
log.Debug(fmt.Sprintf("ref %d", i), "ref", ref)
|
||||
}
|
||||
|
||||
var globalYes, globalNo int
|
||||
var globalMu sync.Mutex
|
||||
var hasErr bool
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(hosts))
|
||||
|
||||
var mu sync.Mutex // mutex protecting the allHostsChunks and bzzAddrs maps
|
||||
allHostChunks := map[string]string{} // host->bitvector of presence for chunks
|
||||
bzzAddrs := map[string]string{} // host->bzzAddr
|
||||
|
||||
for _, host := range hosts {
|
||||
host := host
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
rpcClient, err := rpc.DialContext(ctx, httpHost)
|
||||
if rpcClient != nil {
|
||||
defer rpcClient.Close()
|
||||
}
|
||||
if err != nil {
|
||||
log.Error("error dialing host", "err", err, "host", httpHost)
|
||||
hasErr = true
|
||||
return
|
||||
}
|
||||
|
||||
hostChunks, err := getChunksBitVectorFromHost(rpcClient, addrs)
|
||||
if err != nil {
|
||||
log.Error("error getting chunks bit vector from host", "err", err, "host", httpHost)
|
||||
hasErr = true
|
||||
return
|
||||
}
|
||||
|
||||
bzzAddr, err := getBzzAddrFromHost(rpcClient)
|
||||
if err != nil {
|
||||
log.Error("error getting bzz addrs from host", "err", err, "host", httpHost)
|
||||
hasErr = true
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
allHostChunks[host] = hostChunks
|
||||
bzzAddrs[host] = bzzAddr
|
||||
mu.Unlock()
|
||||
|
||||
yes, no := 0, 0
|
||||
for _, val := range hostChunks {
|
||||
if val == '1' {
|
||||
yes++
|
||||
} else {
|
||||
no++
|
||||
}
|
||||
}
|
||||
|
||||
if no == 0 {
|
||||
log.Info("host reported to have all chunks", "host", host)
|
||||
}
|
||||
|
||||
log.Debug("chunks", "chunks", hostChunks, "yes", yes, "no", no, "host", host)
|
||||
|
||||
if submitMetrics {
|
||||
globalMu.Lock()
|
||||
globalYes += yes
|
||||
globalNo += no
|
||||
globalMu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
checkChunksVsMostProxHosts(addrs, allHostChunks, bzzAddrs)
|
||||
|
||||
if !hasErr && submitMetrics {
|
||||
// remove the chunks stored on the uploader node
|
||||
globalYes -= len(addrs)
|
||||
|
||||
metrics.GetOrRegisterCounter("deployment.chunks.yes", nil).Inc(int64(globalYes))
|
||||
metrics.GetOrRegisterCounter("deployment.chunks.no", nil).Inc(int64(globalNo))
|
||||
metrics.GetOrRegisterCounter("deployment.chunks.refs", nil).Inc(int64(len(addrs)))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getChunksBitVectorFromHost returns a bit vector of presence for a given slice of chunks from a given host
|
||||
func getChunksBitVectorFromHost(client *rpc.Client, addrs []storage.Address) (string, error) {
|
||||
var hostChunks string
|
||||
|
||||
err := client.Call(&hostChunks, "bzz_has", addrs)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return hostChunks, nil
|
||||
}
|
||||
|
||||
// getBzzAddrFromHost returns the bzzAddr for a given host
|
||||
func getBzzAddrFromHost(client *rpc.Client) (string, error) {
|
||||
var hive string
|
||||
|
||||
err := client.Call(&hive, "bzz_hive")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// we make an ugly assumption about the output format of the hive.String() method
|
||||
// ideally we should replace this with an API call that returns the bzz addr for a given host,
|
||||
// but this also works for now (provided we don't change the hive.String() method, which we haven't in some time
|
||||
ss := strings.Split(strings.Split(hive, "\n")[3], " ")
|
||||
return ss[len(ss)-1], nil
|
||||
}
|
||||
|
||||
// checkChunksVsMostProxHosts is checking:
|
||||
// 1. whether a chunk has been found at less than 2 hosts. Considering our NN size, this should not happen.
|
||||
// 2. if a chunk is not found at its closest node. This should also not happen.
|
||||
// Together with the --only-upload flag, we could run this smoke test and make sure that our syncing
|
||||
// functionality is correct (without even trying to retrieve the content).
|
||||
//
|
||||
// addrs - a slice with all uploaded chunk refs
|
||||
// allHostChunks - host->bit vector, showing what chunks are present on what hosts
|
||||
// bzzAddrs - host->bzz address, used when determining the most proximate host for a given chunk
|
||||
func checkChunksVsMostProxHosts(addrs []storage.Address, allHostChunks map[string]string, bzzAddrs map[string]string) {
|
||||
for k, v := range bzzAddrs {
|
||||
log.Trace("bzzAddr", "bzz", v, "host", k)
|
||||
}
|
||||
|
||||
for i := range addrs {
|
||||
var foundAt int
|
||||
maxProx := -1
|
||||
var maxProxHost string
|
||||
for host := range allHostChunks {
|
||||
if allHostChunks[host][i] == '1' {
|
||||
foundAt++
|
||||
}
|
||||
|
||||
ba, err := hex.DecodeString(bzzAddrs[host])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// calculate the host closest to any chunk
|
||||
prox := chunk.Proximity(addrs[i], ba)
|
||||
if prox > maxProx {
|
||||
maxProx = prox
|
||||
maxProxHost = host
|
||||
}
|
||||
}
|
||||
|
||||
if allHostChunks[maxProxHost][i] == '0' {
|
||||
log.Error("chunk not found at max prox host", "ref", addrs[i], "host", maxProxHost, "bzzAddr", bzzAddrs[maxProxHost])
|
||||
} else {
|
||||
log.Trace("chunk present at max prox host", "ref", addrs[i], "host", maxProxHost, "bzzAddr", bzzAddrs[maxProxHost])
|
||||
}
|
||||
|
||||
// if chunk found at less than 2 hosts
|
||||
if foundAt < 2 {
|
||||
log.Error("chunk found at less than two hosts", "foundAt", foundAt, "ref", addrs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getAllRefs(testData []byte) (storage.AddressCollection, error) {
|
||||
datadir, err := ioutil.TempDir("", "chunk-debug")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(datadir)
|
||||
fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32), chunk.NewTags())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(testData)
|
||||
return fileStore.GetAllReferences(context.Background(), reader, false)
|
||||
}
|
||||
|
||||
func uploadAndSync(c *cli.Context, randomBytes []byte) error {
|
||||
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
|
||||
|
||||
t1 := time.Now()
|
||||
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
t2 := time.Since(t1)
|
||||
metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).Update(t2)
|
||||
|
||||
fhash, err := digest(bytes.NewReader(randomBytes))
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("uploaded successfully", "hash", hash, "took", t2, "digest", fmt.Sprintf("%x", fhash))
|
||||
|
||||
// wait to sync and log chunks before fetch attempt, only if syncDelay is set to true
|
||||
if syncDelay {
|
||||
waitToSync()
|
||||
|
||||
log.Debug("chunks before fetch attempt", "hash", hash)
|
||||
|
||||
err = trackChunks(randomBytes, false)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if onlyUpload {
|
||||
log.Debug("only-upload is true, stoppping test", "hash", hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
randIndex := 1 + rand.Intn(len(hosts)-1)
|
||||
|
||||
for {
|
||||
start := time.Now()
|
||||
err := fetch(hash, httpEndpoint(hosts[randIndex]), fhash, "")
|
||||
if err != nil {
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
ended := time.Since(start)
|
||||
|
||||
metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).Update(ended)
|
||||
log.Info("fetch successful", "took", ended, "endpoint", httpEndpoint(hosts[randIndex]))
|
||||
break
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSyncing(wsHost string) (bool, error) {
|
||||
rpcClient, err := rpc.Dial(wsHost)
|
||||
if rpcClient != nil {
|
||||
defer rpcClient.Close()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Error("error dialing host", "err", err)
|
||||
return false, err
|
||||
}
|
||||
|
||||
var isSyncing bool
|
||||
err = rpcClient.Call(&isSyncing, "bzz_isSyncing")
|
||||
if err != nil {
|
||||
log.Error("error calling host for isSyncing", "err", err)
|
||||
return false, err
|
||||
}
|
||||
|
||||
log.Debug("isSyncing result", "host", wsHost, "isSyncing", isSyncing)
|
||||
|
||||
return isSyncing, nil
|
||||
}
|
||||
|
||||
func waitToSync() {
|
||||
t1 := time.Now()
|
||||
|
||||
ns := uint64(1)
|
||||
|
||||
for ns > 0 {
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
notSynced := uint64(0)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(hosts))
|
||||
for i := 0; i < len(hosts); i++ {
|
||||
i := i
|
||||
go func(idx int) {
|
||||
stillSyncing, err := isSyncing(wsEndpoint(hosts[idx]))
|
||||
|
||||
if stillSyncing || err != nil {
|
||||
atomic.AddUint64(¬Synced, 1)
|
||||
}
|
||||
wg.Done()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
ns = atomic.LoadUint64(¬Synced)
|
||||
}
|
||||
|
||||
t2 := time.Since(t1)
|
||||
metrics.GetOrRegisterResettingTimer("upload-and-sync.single.wait-for-sync.deployment", nil).Update(t2)
|
||||
}
|
||||
73
cmd/swarm-smoke/upload_speed.go
Normal file
73
cmd/swarm-smoke/upload_speed.go
Normal file
@@ -0,0 +1,73 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethersphere/swarm/testutil"
|
||||
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
func uploadSpeedCmd(ctx *cli.Context) error {
|
||||
log.Info("uploading to "+hosts[0], "seed", seed)
|
||||
randomBytes := testutil.RandomBytes(seed, filesize*1000)
|
||||
|
||||
errc := make(chan error)
|
||||
|
||||
go func() {
|
||||
errc <- uploadSpeed(ctx, randomBytes)
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-errc:
|
||||
if err != nil {
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
|
||||
}
|
||||
return err
|
||||
case <-time.After(time.Duration(timeout) * time.Second):
|
||||
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
|
||||
|
||||
// trigger debug functionality on randomBytes
|
||||
|
||||
return fmt.Errorf("timeout after %v sec", timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func uploadSpeed(c *cli.Context, data []byte) error {
|
||||
t1 := time.Now()
|
||||
hash, err := upload(data, hosts[0])
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1)))
|
||||
|
||||
fhash, err := digest(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
|
||||
return nil
|
||||
}
|
||||
231
cmd/swarm-smoke/util.go
Normal file
231
cmd/swarm-smoke/util.go
Normal file
@@ -0,0 +1,231 @@
|
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
crand "crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/ethersphere/swarm/api"
|
||||
"github.com/ethersphere/swarm/api/client"
|
||||
"github.com/ethersphere/swarm/spancontext"
|
||||
opentracing "github.com/opentracing/opentracing-go"
|
||||
cli "gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
commandName = ""
|
||||
seed = int(time.Now().UTC().UnixNano())
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(int64(seed))
|
||||
}
|
||||
|
||||
func httpEndpoint(host string) string {
|
||||
return fmt.Sprintf("http://%s:%d", host, httpPort)
|
||||
}
|
||||
|
||||
func wsEndpoint(host string) string {
|
||||
return fmt.Sprintf("ws://%s:%d", host, wsPort)
|
||||
}
|
||||
|
||||
func wrapCliCommand(name string, command func(*cli.Context) error) func(*cli.Context) error {
|
||||
return func(ctx *cli.Context) error {
|
||||
log.PrintOrigins(true)
|
||||
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
|
||||
|
||||
commandName = name
|
||||
|
||||
hosts = strings.Split(allhosts, ",")
|
||||
|
||||
defer func(now time.Time) {
|
||||
totalTime := time.Since(now)
|
||||
log.Info("total time", "time", totalTime, "kb", filesize)
|
||||
metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime)
|
||||
}(time.Now())
|
||||
|
||||
log.Info("smoke test starting", "task", name, "timeout", timeout)
|
||||
metrics.GetOrRegisterCounter(name, nil).Inc(1)
|
||||
|
||||
return command(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
|
||||
ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
|
||||
defer sp.Finish()
|
||||
|
||||
log.Trace("sleeping", "ruid", ruid)
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
|
||||
|
||||
var tn time.Time
|
||||
reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
|
||||
req, _ := http.NewRequest("GET", reqUri, nil)
|
||||
|
||||
opentracing.GlobalTracer().Inject(
|
||||
sp.Context(),
|
||||
opentracing.HTTPHeaders,
|
||||
opentracing.HTTPHeadersCarrier(req.Header))
|
||||
|
||||
trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
|
||||
|
||||
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
|
||||
transport := http.DefaultTransport
|
||||
|
||||
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
|
||||
tn = time.Now()
|
||||
res, err := transport.RoundTrip(req)
|
||||
if err != nil {
|
||||
log.Error(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
|
||||
|
||||
if res.StatusCode != 200 {
|
||||
return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
rdigest, err := digest(res.Body)
|
||||
if err != nil {
|
||||
log.Warn(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
|
||||
if !bytes.Equal(rdigest, original) {
|
||||
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
|
||||
log.Warn(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
|
||||
func fetch(hash string, endpoint string, original []byte, ruid string) error {
|
||||
ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
|
||||
defer sp.Finish()
|
||||
|
||||
log.Info("http get request", "ruid", ruid, "endpoint", endpoint, "hash", hash)
|
||||
|
||||
var tn time.Time
|
||||
reqUri := endpoint + "/bzz:/" + hash + "/"
|
||||
req, _ := http.NewRequest("GET", reqUri, nil)
|
||||
|
||||
opentracing.GlobalTracer().Inject(
|
||||
sp.Context(),
|
||||
opentracing.HTTPHeaders,
|
||||
opentracing.HTTPHeadersCarrier(req.Header))
|
||||
|
||||
trace := client.GetClientTrace(commandName+" - http get", commandName, ruid, &tn)
|
||||
|
||||
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
|
||||
transport := http.DefaultTransport
|
||||
|
||||
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
|
||||
tn = time.Now()
|
||||
res, err := transport.RoundTrip(req)
|
||||
if err != nil {
|
||||
log.Error(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
log.Info("http get response", "ruid", ruid, "endpoint", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
|
||||
|
||||
if res.StatusCode != 200 {
|
||||
err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
|
||||
log.Warn(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
rdigest, err := digest(res.Body)
|
||||
if err != nil {
|
||||
log.Warn(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
|
||||
if !bytes.Equal(rdigest, original) {
|
||||
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
|
||||
log.Warn(err.Error(), "ruid", ruid)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// upload an arbitrary byte as a plaintext file to `endpoint` using the api client
|
||||
func upload(data []byte, endpoint string) (string, error) {
|
||||
swarm := client.NewClient(endpoint)
|
||||
f := &client.File{
|
||||
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
|
||||
ManifestEntry: api.ManifestEntry{
|
||||
ContentType: "text/plain",
|
||||
Mode: 0660,
|
||||
Size: int64(len(data)),
|
||||
},
|
||||
}
|
||||
|
||||
// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
|
||||
return swarm.Upload(f, "", false)
|
||||
}
|
||||
|
||||
func digest(r io.Reader) ([]byte, error) {
|
||||
h := md5.New()
|
||||
_, err := io.Copy(h, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.Sum(nil), nil
|
||||
}
|
||||
|
||||
// generates random data in heap buffer
|
||||
func generateRandomData(datasize int) ([]byte, error) {
|
||||
b := make([]byte, datasize)
|
||||
c, err := crand.Read(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if c != datasize {
|
||||
return nil, errors.New("short read")
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
Reference in New Issue
Block a user