cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)
This adds a swarm ls command which lists files and directories stored in a manifest. Rather than listing all files, it uses "directory prefixes" in case there are a lot of files in a manifest but you just want to traverse it. This also includes some refactoring to the tests and the introduction of a swarm/api/client package to make things easier to test.
This commit is contained in:
committed by
Felix Lange
parent
09777952ee
commit
b319f027a0
58
cmd/swarm/list.go
Normal file
58
cmd/swarm/list.go
Normal file
@ -0,0 +1,58 @@
|
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
|
||||
"gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
func list(ctx *cli.Context) {
|
||||
args := ctx.Args()
|
||||
|
||||
if len(args) < 1 {
|
||||
utils.Fatalf("Please supply a manifest reference as the first argument")
|
||||
} else if len(args) > 2 {
|
||||
utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
|
||||
}
|
||||
manifest := args[0]
|
||||
|
||||
var prefix string
|
||||
if len(args) == 2 {
|
||||
prefix = args[1]
|
||||
}
|
||||
|
||||
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
|
||||
client := swarm.NewClient(bzzapi)
|
||||
entries, err := client.ManifestFileList(manifest, prefix)
|
||||
if err != nil {
|
||||
utils.Fatalf("Failed to generate file and directory list: %s", err)
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
|
||||
defer w.Flush()
|
||||
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
|
||||
for _, entry := range entries {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
|
||||
}
|
||||
}
|
@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
|
||||
ArgsUsage: " <file>",
|
||||
Description: `
|
||||
"upload a file or directory to swarm using the HTTP API and prints the root hash",
|
||||
`,
|
||||
},
|
||||
{
|
||||
Action: list,
|
||||
Name: "ls",
|
||||
Usage: "list files and directories contained in a manifest",
|
||||
ArgsUsage: " <manifest> [<prefix>]",
|
||||
Description: `
|
||||
Lists files and directories contained in a manifest.
|
||||
`,
|
||||
},
|
||||
{
|
||||
|
@ -25,6 +25,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
|
||||
"gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
|
||||
|
||||
ctype string
|
||||
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
|
||||
mroot manifest
|
||||
mroot swarm.Manifest
|
||||
)
|
||||
|
||||
if len(args) > 3 {
|
||||
@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
|
||||
|
||||
ctype string
|
||||
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
|
||||
mroot manifest
|
||||
mroot swarm.Manifest
|
||||
)
|
||||
if len(args) > 3 {
|
||||
ctype = args[3]
|
||||
@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
|
||||
path = args[1]
|
||||
|
||||
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
|
||||
mroot manifest
|
||||
mroot swarm.Manifest
|
||||
)
|
||||
|
||||
newManifest := removeEntryFromManifest(ctx, mhash, path)
|
||||
@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
|
||||
|
||||
var (
|
||||
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
|
||||
client = &client{api: bzzapi}
|
||||
longestPathEntry = manifestEntry{
|
||||
client = swarm.NewClient(bzzapi)
|
||||
longestPathEntry = swarm.ManifestEntry{
|
||||
Path: "",
|
||||
Hash: "",
|
||||
ContentType: "",
|
||||
}
|
||||
)
|
||||
|
||||
mroot, err := client.downloadManifest(mhash)
|
||||
mroot, err := client.DownloadManifest(mhash)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest download failed: %v", err)
|
||||
}
|
||||
|
||||
//TODO: check if the "hash" to add is valid and present in swarm
|
||||
_, err = client.downloadManifest(hash)
|
||||
_, err = client.DownloadManifest(hash)
|
||||
if err != nil {
|
||||
utils.Fatalf("Hash to add is not present: %v", err)
|
||||
}
|
||||
@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
|
||||
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
|
||||
|
||||
// Replace the hash for parent Manifests
|
||||
newMRoot := manifest{}
|
||||
newMRoot := swarm.Manifest{}
|
||||
for _, entry := range mroot.Entries {
|
||||
if longestPathEntry.Path == entry.Path {
|
||||
entry.Hash = newHash
|
||||
@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
|
||||
mroot = newMRoot
|
||||
} else {
|
||||
// Add the entry in the leaf Manifest
|
||||
newEntry := manifestEntry{
|
||||
newEntry := swarm.ManifestEntry{
|
||||
Path: path,
|
||||
Hash: hash,
|
||||
ContentType: ctype,
|
||||
@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
|
||||
mroot.Entries = append(mroot.Entries, newEntry)
|
||||
}
|
||||
|
||||
newManifestHash, err := client.uploadManifest(mroot)
|
||||
newManifestHash, err := client.UploadManifest(mroot)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest upload failed: %v", err)
|
||||
}
|
||||
@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
|
||||
|
||||
var (
|
||||
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
|
||||
client = &client{api: bzzapi}
|
||||
newEntry = manifestEntry{
|
||||
client = swarm.NewClient(bzzapi)
|
||||
newEntry = swarm.ManifestEntry{
|
||||
Path: "",
|
||||
Hash: "",
|
||||
ContentType: "",
|
||||
}
|
||||
longestPathEntry = manifestEntry{
|
||||
longestPathEntry = swarm.ManifestEntry{
|
||||
Path: "",
|
||||
Hash: "",
|
||||
ContentType: "",
|
||||
}
|
||||
)
|
||||
|
||||
mroot, err := client.downloadManifest(mhash)
|
||||
mroot, err := client.DownloadManifest(mhash)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest download failed: %v", err)
|
||||
}
|
||||
@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
|
||||
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
|
||||
|
||||
// Replace the hash for parent Manifests
|
||||
newMRoot := manifest{}
|
||||
newMRoot := swarm.Manifest{}
|
||||
for _, entry := range mroot.Entries {
|
||||
if longestPathEntry.Path == entry.Path {
|
||||
entry.Hash = newHash
|
||||
@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
|
||||
|
||||
if newEntry.Path != "" {
|
||||
// Replace the hash for leaf Manifest
|
||||
newMRoot := manifest{}
|
||||
newMRoot := swarm.Manifest{}
|
||||
for _, entry := range mroot.Entries {
|
||||
if newEntry.Path == entry.Path {
|
||||
myEntry := manifestEntry{
|
||||
myEntry := swarm.ManifestEntry{
|
||||
Path: entry.Path,
|
||||
Hash: hash,
|
||||
ContentType: ctype,
|
||||
@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
|
||||
mroot = newMRoot
|
||||
}
|
||||
|
||||
newManifestHash, err := client.uploadManifest(mroot)
|
||||
newManifestHash, err := client.UploadManifest(mroot)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest upload failed: %v", err)
|
||||
}
|
||||
@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
|
||||
|
||||
var (
|
||||
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
|
||||
client = &client{api: bzzapi}
|
||||
entryToRemove = manifestEntry{
|
||||
client = swarm.NewClient(bzzapi)
|
||||
entryToRemove = swarm.ManifestEntry{
|
||||
Path: "",
|
||||
Hash: "",
|
||||
ContentType: "",
|
||||
}
|
||||
longestPathEntry = manifestEntry{
|
||||
longestPathEntry = swarm.ManifestEntry{
|
||||
Path: "",
|
||||
Hash: "",
|
||||
ContentType: "",
|
||||
}
|
||||
)
|
||||
|
||||
mroot, err := client.downloadManifest(mhash)
|
||||
mroot, err := client.DownloadManifest(mhash)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest download failed: %v", err)
|
||||
}
|
||||
@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
|
||||
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
|
||||
|
||||
// Replace the hash for parent Manifests
|
||||
newMRoot := manifest{}
|
||||
newMRoot := swarm.Manifest{}
|
||||
for _, entry := range mroot.Entries {
|
||||
if longestPathEntry.Path == entry.Path {
|
||||
entry.Hash = newHash
|
||||
@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
|
||||
|
||||
if entryToRemove.Path != "" {
|
||||
// remove the entry in this Manifest
|
||||
newMRoot := manifest{}
|
||||
newMRoot := swarm.Manifest{}
|
||||
for _, entry := range mroot.Entries {
|
||||
if entryToRemove.Path != entry.Path {
|
||||
newMRoot.Entries = append(newMRoot.Entries, entry)
|
||||
@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
|
||||
mroot = newMRoot
|
||||
}
|
||||
|
||||
newManifestHash, err := client.uploadManifest(mroot)
|
||||
newManifestHash, err := client.UploadManifest(mroot)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest upload failed: %v", err)
|
||||
}
|
||||
|
@ -18,21 +18,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
|
||||
"gopkg.in/urfave/cli.v1"
|
||||
)
|
||||
|
||||
@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
|
||||
|
||||
var (
|
||||
file = args[0]
|
||||
client = &client{api: bzzapi}
|
||||
client = swarm.NewClient(bzzapi)
|
||||
)
|
||||
fi, err := os.Stat(expandPath(file))
|
||||
if err != nil {
|
||||
@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
|
||||
if !wantManifest {
|
||||
utils.Fatalf("Manifest is required for directory uploads")
|
||||
}
|
||||
mhash, err := client.uploadDirectory(file, defaultPath)
|
||||
mhash, err := client.UploadDirectory(file, defaultPath)
|
||||
if err != nil {
|
||||
utils.Fatalf("Failed to upload directory: %v", err)
|
||||
}
|
||||
fmt.Println(mhash)
|
||||
return
|
||||
}
|
||||
entry, err := client.uploadFile(file, fi)
|
||||
entry, err := client.UploadFile(file, fi)
|
||||
if err != nil {
|
||||
utils.Fatalf("Upload failed: %v", err)
|
||||
}
|
||||
mroot := manifest{[]manifestEntry{entry}}
|
||||
mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
|
||||
if !wantManifest {
|
||||
// Print the manifest. This is the only output to stdout.
|
||||
mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
|
||||
fmt.Println(string(mrootJSON))
|
||||
return
|
||||
}
|
||||
hash, err := client.uploadManifest(mroot)
|
||||
hash, err := client.UploadManifest(mroot)
|
||||
if err != nil {
|
||||
utils.Fatalf("Manifest upload failed: %v", err)
|
||||
}
|
||||
@ -111,148 +105,3 @@ func homeDir() string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// client wraps interaction with the swarm HTTP gateway.
|
||||
type client struct {
|
||||
api string
|
||||
}
|
||||
|
||||
// manifest is the JSON representation of a swarm manifest.
|
||||
type manifestEntry struct {
|
||||
Hash string `json:"hash,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
// manifest is the JSON representation of a swarm manifest.
|
||||
type manifest struct {
|
||||
Entries []manifestEntry `json:"entries,omitempty"`
|
||||
}
|
||||
|
||||
func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
|
||||
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload empty manifest")
|
||||
}
|
||||
if len(defaultPath) > 0 {
|
||||
fi, err := os.Stat(defaultPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
|
||||
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
|
||||
if err != nil || fi.IsDir() {
|
||||
return err
|
||||
}
|
||||
if !strings.HasPrefix(path, dir) {
|
||||
return fmt.Errorf("path %s outside directory %s", path, dir)
|
||||
}
|
||||
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
|
||||
mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
|
||||
return err
|
||||
})
|
||||
return mhash, err
|
||||
}
|
||||
|
||||
func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
|
||||
hash, err := c.uploadFileContent(file, fi)
|
||||
m := manifestEntry{
|
||||
Hash: hash,
|
||||
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
|
||||
fd, err := os.Open(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer fd.Close()
|
||||
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
|
||||
return c.postRaw("application/octet-stream", fi.Size(), fd)
|
||||
}
|
||||
|
||||
func (c *client) uploadManifest(m manifest) (string, error) {
|
||||
jsm, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.Info("Uploading swarm manifest")
|
||||
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
|
||||
}
|
||||
|
||||
func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
|
||||
fd, err := os.Open(fpath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer fd.Close()
|
||||
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
|
||||
req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
|
||||
req.ContentLength = fi.Size()
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return "", fmt.Errorf("bad status: %s", resp.Status)
|
||||
}
|
||||
content, err := ioutil.ReadAll(resp.Body)
|
||||
return string(content), err
|
||||
}
|
||||
|
||||
func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
|
||||
req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("content-type", mimetype)
|
||||
req.ContentLength = size
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return "", fmt.Errorf("bad status: %s", resp.Status)
|
||||
}
|
||||
content, err := ioutil.ReadAll(resp.Body)
|
||||
return string(content), err
|
||||
}
|
||||
|
||||
func (c *client) downloadManifest(mhash string) (manifest, error) {
|
||||
|
||||
mroot := manifest{}
|
||||
req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
|
||||
if err != nil {
|
||||
return mroot, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return mroot, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return mroot, fmt.Errorf("bad status: %s", resp.Status)
|
||||
|
||||
}
|
||||
content, err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
err = json.Unmarshal(content, &mroot)
|
||||
if err != nil {
|
||||
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
|
||||
}
|
||||
return mroot, err
|
||||
}
|
||||
|
Reference in New Issue
Block a user