cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.
This commit is contained in:
Lewis Marshall
2017-04-04 23:20:07 +01:00
committed by Felix Lange
parent 09777952ee
commit b319f027a0
10 changed files with 594 additions and 245 deletions

58
cmd/swarm/list.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"strings"
"text/tabwriter"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
func list(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Please supply a manifest reference as the first argument")
} else if len(args) > 2 {
utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
}
manifest := args[0]
var prefix string
if len(args) == 2 {
prefix = args[1]
}
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
entries, err := client.ManifestFileList(manifest, prefix)
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
for _, entry := range entries {
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
}
}

View File

@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
ArgsUsage: " <file>",
Description: `
"upload a file or directory to swarm using the HTTP API and prints the root hash",
`,
},
{
Action: list,
Name: "ls",
Usage: "list files and directories contained in a manifest",
ArgsUsage: " <manifest> [<prefix>]",
Description: `
Lists files and directories contained in a manifest.
`,
},
{

View File

@ -25,6 +25,7 @@ import (
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest
mroot swarm.Manifest
)
if len(args) > 3 {
@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest
mroot swarm.Manifest
)
if len(args) > 3 {
ctype = args[3]
@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest
mroot swarm.Manifest
)
newManifest := removeEntryFromManifest(ctx, mhash, path)
@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi}
longestPathEntry = manifestEntry{
client = swarm.NewClient(bzzapi)
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
mroot, err := client.downloadManifest(mhash)
mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
//TODO: check if the "hash" to add is valid and present in swarm
_, err = client.downloadManifest(hash)
_, err = client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Hash to add is not present: %v", err)
}
@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
newEntry := manifestEntry{
newEntry := swarm.ManifestEntry{
Path: path,
Hash: hash,
ContentType: ctype,
@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot.Entries = append(mroot.Entries, newEntry)
}
newManifestHash, err := client.uploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi}
newEntry = manifestEntry{
client = swarm.NewClient(bzzapi)
newEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = manifestEntry{
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
mroot, err := client.downloadManifest(mhash)
mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if newEntry.Path != "" {
// Replace the hash for leaf Manifest
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path {
myEntry := manifestEntry{
myEntry := swarm.ManifestEntry{
Path: entry.Path,
Hash: hash,
ContentType: ctype,
@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
mroot = newMRoot
}
newManifestHash, err := client.uploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi}
entryToRemove = manifestEntry{
client = swarm.NewClient(bzzapi)
entryToRemove = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
longestPathEntry = manifestEntry{
longestPathEntry = swarm.ManifestEntry{
Path: "",
Hash: "",
ContentType: "",
}
)
mroot, err := client.downloadManifest(mhash)
mroot, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if entryToRemove.Path != "" {
// remove the entry in this Manifest
newMRoot := manifest{}
newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry)
@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
mroot = newMRoot
}
newManifestHash, err := client.uploadManifest(mroot)
newManifestHash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}

View File

@ -18,21 +18,15 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
"os/user"
"path"
"path/filepath"
"strings"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
var (
file = args[0]
client = &client{api: bzzapi}
client = swarm.NewClient(bzzapi)
)
fi, err := os.Stat(expandPath(file))
if err != nil {
@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
if !wantManifest {
utils.Fatalf("Manifest is required for directory uploads")
}
mhash, err := client.uploadDirectory(file, defaultPath)
mhash, err := client.UploadDirectory(file, defaultPath)
if err != nil {
utils.Fatalf("Failed to upload directory: %v", err)
}
fmt.Println(mhash)
return
}
entry, err := client.uploadFile(file, fi)
entry, err := client.UploadFile(file, fi)
if err != nil {
utils.Fatalf("Upload failed: %v", err)
}
mroot := manifest{[]manifestEntry{entry}}
mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
if !wantManifest {
// Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON))
return
}
hash, err := client.uploadManifest(mroot)
hash, err := client.UploadManifest(mroot)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
@ -111,148 +105,3 @@ func homeDir() string {
}
return ""
}
// client wraps interaction with the swarm HTTP gateway.
type client struct {
api string
}
// manifest is the JSON representation of a swarm manifest.
type manifestEntry struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
}
// manifest is the JSON representation of a swarm manifest.
type manifest struct {
Entries []manifestEntry `json:"entries,omitempty"`
}
func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
if err != nil {
return "", fmt.Errorf("failed to upload empty manifest")
}
if len(defaultPath) > 0 {
fi, err := os.Stat(defaultPath)
if err != nil {
return "", err
}
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
if err != nil {
return "", err
}
}
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() {
return err
}
if !strings.HasPrefix(path, dir) {
return fmt.Errorf("path %s outside directory %s", path, dir)
}
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
return err
})
return mhash, err
}
func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
hash, err := c.uploadFileContent(file, fi)
m := manifestEntry{
Hash: hash,
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
}
return m, err
}
func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
fd, err := os.Open(file)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
return c.postRaw("application/octet-stream", fi.Size(), fd)
}
func (c *client) uploadManifest(m manifest) (string, error) {
jsm, err := json.Marshal(m)
if err != nil {
panic(err)
}
log.Info("Uploading swarm manifest")
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
}
func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
fd, err := os.Open(fpath)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
if err != nil {
return "", err
}
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
req.ContentLength = fi.Size()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
if err != nil {
return "", err
}
req.Header.Set("content-type", mimetype)
req.ContentLength = size
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *client) downloadManifest(mhash string) (manifest, error) {
mroot := manifest{}
req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
if err != nil {
return mroot, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return mroot, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return mroot, fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(content, &mroot)
if err != nil {
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
}
return mroot, err
}