gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

Browse Source 
This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.
This commit is contained in:
Lewis Marshall
2017-04-04 23:20:07 +01:00
committed by Felix Lange
parent 09777952ee
commit b319f027a0
10 changed files with 594 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
cmd/swarm/list.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"strings"
"text/tabwriter"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
func list(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Please supply a manifest reference as the first argument")
} else if len(args) > 2 {
utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
}
manifest := args[0]
var prefix string
if len(args) == 2 {
prefix = args[1]
}
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
entries, err := client.ManifestFileList(manifest, prefix)
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
for _, entry := range entries {
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
}
}

9
cmd/swarm/main.go
View File

@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
ArgsUsage: " <file>",
Description: `
"upload a file or directory to swarm using the HTTP API and prints the root hash",
`,
},
{
Action: list,
Name: "ls",
Usage: "list files and directories contained in a manifest",
ArgsUsage: " <manifest> [<prefix>]",
Description: `
Lists files and directories contained in a manifest.
`,
},
{

51
cmd/swarm/manifest.go
View File

@ -25,6 +25,7 @@ import (
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest
mroot swarm.Manifest
)
if len(args) > 3 {
@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest
mroot swarm.Manifest
)
if len(args) > 3 {
ctype = args[3]
@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest
mroot swarm.Manifest
)
newManifest := removeEntryFromManifest(ctx, mhash, path)
@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi}
longestPathEntry = manifestEntry{
client = swarm.NewClient(bzzapi)
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
mroot, err := client.downloadManifest(mhash)
mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
//TODO: check if the "hash" to add is valid and present in swarm
_, err = client.downloadManifest(hash)
_, err = client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Hash to add is not present: %v", err)
}
@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
newEntry := manifestEntry{
newEntry := swarm.ManifestEntry{
Path: path,
Hash: hash,
ContentType: ctype,
@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot.Entries = append(mroot.Entries, newEntry)
}
newManifestHash, err := client.uploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi}
newEntry = manifestEntry{
client = swarm.NewClient(bzzapi)
newEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = manifestEntry{
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
mroot, err := client.downloadManifest(mhash)
mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if newEntry.Path != "" {
// Replace the hash for leaf Manifest
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path {
myEntry := manifestEntry{
myEntry := swarm.ManifestEntry{
Path: entry.Path,
Hash: hash,
ContentType: ctype,
@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
mroot = newMRoot
}
newManifestHash, err := client.uploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi}
entryToRemove = manifestEntry{
client = swarm.NewClient(bzzapi)
entryToRemove = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = manifestEntry{
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
mroot, err := client.downloadManifest(mhash)
mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if entryToRemove.Path != "" {
// remove the entry in this Manifest
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry)
@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
mroot = newMRoot
}
newManifestHash, err := client.uploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}

163
cmd/swarm/upload.go
View File

@ -18,21 +18,15 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
"os/user"
"path"
"path/filepath"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
var (
file = args[0]
client = &client{api: bzzapi}
client = swarm.NewClient(bzzapi)
)
fi, err := os.Stat(expandPath(file))
if err != nil {
@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
if !wantManifest {
utils.Fatalf("Manifest is required for directory uploads")
}
mhash, err := client.uploadDirectory(file, defaultPath)
mhash, err := client.UploadDirectory(file, defaultPath)
if err != nil {
utils.Fatalf("Failed to upload directory: %v", err)
}
fmt.Println(mhash)
return
}
entry, err := client.uploadFile(file, fi)
entry, err := client.UploadFile(file, fi)
if err != nil {
utils.Fatalf("Upload failed: %v", err)
}
mroot := manifest{[]manifestEntry{entry}}
mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
hash, err := client.uploadManifest(mroot)
hash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@ -111,148 +105,3 @@ func homeDir() string {
}
return ""
}
// client wraps interaction with the swarm HTTP gateway.
type client struct {
api string
}
// manifest is the JSON representation of a swarm manifest.
type manifestEntry struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
}
// manifest is the JSON representation of a swarm manifest.
type manifest struct {
Entries []manifestEntry `json:"entries,omitempty"`
}
func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
if err != nil {
return "", fmt.Errorf("failed to upload empty manifest")
}
if len(defaultPath) > 0 {
fi, err := os.Stat(defaultPath)
if err != nil {
return "", err
}
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
if err != nil {
return "", err
}
}
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() {
return err
}
if !strings.HasPrefix(path, dir) {
return fmt.Errorf("path %s outside directory %s", path, dir)
}
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
return err
})
return mhash, err
}
func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
hash, err := c.uploadFileContent(file, fi)
m := manifestEntry{
Hash: hash,
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
}
return m, err
}
func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
fd, err := os.Open(file)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
return c.postRaw("application/octet-stream", fi.Size(), fd)
}
func (c *client) uploadManifest(m manifest) (string, error) {
jsm, err := json.Marshal(m)
if err != nil {
panic(err)
}
log.Info("Uploading swarm manifest")
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
}
func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
fd, err := os.Open(fpath)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
if err != nil {
return "", err
}
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
req.ContentLength = fi.Size()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
if err != nil {
return "", err
}
req.Header.Set("content-type", mimetype)
req.ContentLength = size
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *client) downloadManifest(mhash string) (manifest, error) {
mroot := manifest{}
req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
if err != nil {
return mroot, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return mroot, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return mroot, fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(content, &mroot)
if err != nil {
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
}
return mroot, err
}