gaspersic/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

swarm/api: refactor and improve HTTP API (#3773)

Browse Source 
This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).
This commit is contained in:
Lewis Marshall
2017-04-06 23:22:22 +01:00
committed by Felix Lange
parent 9aca9e6deb
commit 71fdaa4238
20 changed files with 1822 additions and 706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
swarm/api/manifest.go
View File

@ -19,8 +19,11 @@ package api
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
@ -28,25 +31,152 @@ import (
)
const (
manifestType = "application/bzz-manifest+json"
ManifestType = "application/bzz-manifest+json"
)
// Manifest represents a swarm manifest
type Manifest struct {
Entries []ManifestEntry `json:"entries,omitempty"`
}
// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
}
// ManifestList represents the result of listing files in a manifest
type ManifestList struct {
CommonPrefixes []string `json:"common_prefixes,omitempty"`
Entries []*ManifestEntry `json:"entries,omitempty"`
}
// NewManifest creates and stores a new, empty manifest
func (a *Api) NewManifest() (storage.Key, error) {
var manifest Manifest
data, err := json.Marshal(&manifest)
if err != nil {
return nil, err
}
return a.Store(bytes.NewReader(data), int64(len(data)), nil)
}
// ManifestWriter is used to add and remove entries from an underlying manifest
type ManifestWriter struct {
api *Api
trie *manifestTrie
quitC chan bool
}
func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWriter, error) {
trie, err := loadManifest(a.dpa, key, quitC)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
}
return &ManifestWriter{a, trie, quitC}, nil
}
// AddEntry stores the given data and adds the resulting key to the manifest
func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
key, err := m.api.Store(data, e.Size, nil)
if err != nil {
return nil, err
}
entry := newManifestTrieEntry(e, nil)
entry.Hash = key.String()
m.trie.addEntry(entry, m.quitC)
return key, nil
}
// RemoveEntry removes the given path from the manifest
func (m *ManifestWriter) RemoveEntry(path string) error {
m.trie.deleteEntry(path, m.quitC)
return nil
}
// Store stores the manifest, returning the resulting storage key
func (m *ManifestWriter) Store() (storage.Key, error) {
return m.trie.hash, m.trie.recalcAndStore()
}
// ManifestWalker is used to recursively walk the entries in the manifest and
// all of its submanifests
type ManifestWalker struct {
api *Api
trie *manifestTrie
quitC chan bool
}
func (a *Api) NewManifestWalker(key storage.Key, quitC chan bool) (*ManifestWalker, error) {
trie, err := loadManifest(a.dpa, key, quitC)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", key, err)
}
return &ManifestWalker{a, trie, quitC}, nil
}
// SkipManifest is used as a return value from WalkFn to indicate that the
// manifest should be skipped
var SkipManifest = errors.New("skip this manifest")
// WalkFn is the type of function called for each entry visited by a recursive
// manifest walk
type WalkFn func(entry *ManifestEntry) error
// Walk recursively walks the manifest calling walkFn for each entry in the
// manifest, including submanifests
func (m *ManifestWalker) Walk(walkFn WalkFn) error {
return m.walk(m.trie, "", walkFn)
}
func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
for _, entry := range trie.entries {
if entry == nil {
continue
}
entry.Path = prefix + entry.Path
err := walkFn(&entry.ManifestEntry)
if err != nil {
if entry.ContentType == ManifestType && err == SkipManifest {
continue
}
return err
}
if entry.ContentType != ManifestType {
continue
}
if err := trie.loadSubTrie(entry, nil); err != nil {
return err
}
if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
return err
}
}
return nil
}
type manifestTrie struct {
dpa *storage.DPA
entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
hash storage.Key // if hash != nil, it is stored
}
type manifestJSON struct {
Entries []*manifestTrieEntry `json:"entries"`
func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
return &manifestTrieEntry{
ManifestEntry: *entry,
subtrie: subtrie,
}
}
type manifestTrieEntry struct {
Path string `json:"path"`
Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
ContentType string `json:"contentType"`
Status int `json:"status"`
subtrie *manifestTrie
ManifestEntry
subtrie *manifestTrie
}
func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
@ -77,7 +207,9 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
}
log.Trace(fmt.Sprintf("Manifest %v retrieved", hash.Log()))
man := manifestJSON{}
var man struct {
Entries []*manifestTrieEntry `json:"entries"`
}
err = json.Unmarshal(manifestData, &man)
if err != nil {
err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err)
@ -116,7 +248,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
cpl++
}
if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if self.loadSubTrie(oldentry, quitC) != nil {
return
}
@ -136,12 +268,10 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
self.entries[b] = &manifestTrieEntry{
self.entries[b] = newManifestTrieEntry(&ManifestEntry{
Path: commonPrefix,
Hash: "",
ContentType: manifestType,
subtrie: subtrie,
}
ContentType: ManifestType,
}, subtrie)
}
func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
@ -173,7 +303,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
}
epl := len(entry.Path)
if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if self.loadSubTrie(entry, quitC) != nil {
return
}
@ -198,7 +328,7 @@ func (self *manifestTrie) recalcAndStore() error {
var buffer bytes.Buffer
buffer.WriteString(`{"entries":[`)
list := &manifestJSON{}
list := &Manifest{}
for _, entry := range self.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
@ -208,7 +338,7 @@ func (self *manifestTrie) recalcAndStore() error {
}
entry.Hash = entry.subtrie.hash.String()
}
list.Entries = append(list.Entries, entry)
list.Entries = append(list.Entries, entry.ManifestEntry)
}
}
@ -254,7 +384,7 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool,
entry := self.entries[i]
if entry != nil {
epl := len(entry.Path)
if entry.ContentType == manifestType {
if entry.ContentType == ManifestType {
l := plen
if epl < l {
l = epl
@ -300,7 +430,7 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl))
if (len(path) >= epl) && (path[:epl] == entry.Path) {
log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
if entry.ContentType == manifestType {
if entry.ContentType == ManifestType {
err := self.loadSubTrie(entry, quitC)
if err != nil {
return nil, 0