swarm: codebase split from go-ethereum (#1405)

api/http/middleware.go

@@ -0,0 +1,162 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+	"runtime/debug"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethersphere/swarm/api"
+	"github.com/ethersphere/swarm/chunk"
+	"github.com/ethersphere/swarm/log"
+	"github.com/ethersphere/swarm/sctx"
+	"github.com/ethersphere/swarm/spancontext"
+	"github.com/pborman/uuid"
+)
+
+// Adapt chains h (main request handler) main handler to adapters (middleware handlers)
+// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first)
+func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
+	for i := range adapters {
+		adapter := adapters[len(adapters)-1-i]
+		h = adapter(h)
+	}
+	return h
+}
+
+type Adapter func(http.Handler) http.Handler
+
+func SetRequestID(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8]))
+		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
+		log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func SetRequestHost(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r = r.WithContext(sctx.SetHost(r.Context(), r.Host))
+		log.Info("setting request host", "ruid", GetRUID(r.Context()), "host", sctx.GetHost(r.Context()))
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func ParseURI(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			respondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
+			return
+		}
+		if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") {
+			uri.Addr = strings.TrimPrefix(uri.Addr, "0x")
+
+			msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
+			Please click <a href='%[1]s'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uri.String())
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(msg))
+			return
+		}
+
+		ctx := r.Context()
+		r = r.WithContext(SetURI(ctx, uri))
+		log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func InitLoggingResponseWriter(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tn := time.Now()
+
+		writer := newLoggingResponseWriter(w)
+		h.ServeHTTP(writer, r)
+
+		ts := time.Since(tn)
+		log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode, "time", ts)
+		metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).Update(ts)
+		metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).Update(ts)
+	})
+}
+
+// InitUploadTag creates a new tag for an upload to the local HTTP proxy
+// if a tag is not named using the SwarmTagHeaderName, a fallback name will be used
+// when the Content-Length header is set, an ETA on chunking will be available since the
+// number of chunks to be split is known in advance (not including enclosing manifest chunks)
+// the tag can later be accessed using the appropriate identifier in the request context
+func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var (
+			tagName        string
+			err            error
+			estimatedTotal int64 = 0
+			contentType          = r.Header.Get("Content-Type")
+			headerTag            = r.Header.Get(SwarmTagHeaderName)
+		)
+		if headerTag != "" {
+			tagName = headerTag
+			log.Trace("got tag name from http header", "tagName", tagName)
+		} else {
+			tagName = fmt.Sprintf("unnamed_tag_%d", time.Now().Unix())
+		}
+
+		if !strings.Contains(contentType, "multipart") && r.ContentLength > 0 {
+			log.Trace("calculating tag size", "contentType", contentType, "contentLength", r.ContentLength)
+			uri := GetURI(r.Context())
+			if uri != nil {
+				log.Debug("got uri from context")
+				if uri.Addr == "encrypt" {
+					estimatedTotal = calculateNumberOfChunks(r.ContentLength, true)
+				} else {
+					estimatedTotal = calculateNumberOfChunks(r.ContentLength, false)
+				}
+			}
+		}
+
+		log.Trace("creating tag", "tagName", tagName, "estimatedTotal", estimatedTotal)
+
+		t, err := tags.New(tagName, estimatedTotal)
+		if err != nil {
+			log.Error("error creating tag", "err", err, "tagName", tagName)
+		}
+
+		log.Trace("setting tag id to context", "uid", t.Uid)
+		ctx := sctx.SetTag(r.Context(), t.Uid)
+
+		h.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func InstrumentOpenTracing(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri := GetURI(r.Context())
+		if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
+			h.ServeHTTP(w, r) // soft fail
+			return
+		}
+		spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme)
+		ctx, sp := spancontext.StartSpan(r.Context(), spanName)
+
+		defer sp.Finish()
+		h.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func RecoverPanic(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Error("panic recovery!", "stack trace", string(debug.Stack()), "url", r.URL.String(), "headers", r.Header)
+			}
+		}()
+		h.ServeHTTP(w, r)
+	})
+}

api/http/response.go

@@ -0,0 +1,132 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethersphere/swarm/api"
+)
+
+var (
+	htmlCounter      = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
+	jsonCounter      = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
+	plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil)
+)
+
+type ResponseParams struct {
+	Msg       template.HTML
+	Code      int
+	Timestamp string
+	template  *template.Template
+	Details   template.HTML
+}
+
+// ShowMultipleChoices is used when a user requests a resource in a manifest which results
+// in ambiguous results. It returns a HTML page with clickable links of each of the entry
+// in the manifest which fits the request URI ambiguity.
+// For example, if the user requests bzz:/<hash>/read and that manifest contains entries
+// "readme.md" and "readinglist.txt", a HTML page is returned with this two links.
+// This only applies if the manifest has no default entry
+func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
+	log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	msg := ""
+	if list.Entries == nil {
+		respondError(w, r, "Could not resolve", http.StatusInternalServerError)
+		return
+	}
+	requestUri := strings.TrimPrefix(r.RequestURI, "/")
+
+	uri, err := api.Parse(requestUri)
+	if err != nil {
+		respondError(w, r, "Bad Request", http.StatusBadRequest)
+	}
+
+	uri.Scheme = "bzz-list"
+	msg += fmt.Sprintf("Disambiguation:<br/>Your request may refer to multiple choices.<br/>Click <a class=\"orange\" href='"+"/"+uri.String()+"'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout(\"location.href='%s';\",5000);</script><br/>", "/"+uri.String())
+	respondTemplate(w, r, "error", msg, http.StatusMultipleChoices)
+}
+
+func respondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) {
+	log.Debug("respondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	respond(w, r, &ResponseParams{
+		Code:      code,
+		Msg:       template.HTML(msg),
+		Timestamp: time.Now().Format(time.RFC1123),
+		template:  TemplatesMap[templateName],
+	})
+}
+
+func respondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
+	log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code, "msg", msg)
+	respondTemplate(w, r, "error", msg, code)
+}
+
+func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
+	w.WriteHeader(params.Code)
+
+	if params.Code >= 400 {
+		w.Header().Del("Cache-Control")
+		w.Header().Del("ETag")
+	}
+
+	acceptHeader := r.Header.Get("Accept")
+	// this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8"
+	if strings.Contains(acceptHeader, "application/json") {
+		if err := respondJSON(w, r, params); err != nil {
+			respondError(w, r, "Internal server error", http.StatusInternalServerError)
+		}
+	} else if strings.Contains(acceptHeader, "text/html") {
+		respondHTML(w, r, params)
+	} else {
+		respondPlaintext(w, r, params) //returns nice errors for curl
+	}
+}
+
+func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
+	htmlCounter.Inc(1)
+	log.Info("respondHTML", "ruid", GetRUID(r.Context()), "code", params.Code)
+	err := params.template.Execute(w, params)
+	if err != nil {
+		log.Error(err.Error())
+	}
+}
+
+func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	jsonCounter.Inc(1)
+	log.Info("respondJSON", "ruid", GetRUID(r.Context()), "code", params.Code)
+	w.Header().Set("Content-Type", "application/json")
+	return json.NewEncoder(w).Encode(params)
+}
+
+func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	plaintextCounter.Inc(1)
+	log.Info("respondPlaintext", "ruid", GetRUID(r.Context()), "code", params.Code)
+	w.Header().Set("Content-Type", "text/plain")
+	strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n"
+	strToWrite += "Message: " + string(params.Msg) + "\n"
+	strToWrite += "Timestamp: " + params.Timestamp + "\n"
+	_, err := w.Write([]byte(strToWrite))
+	return err
+}

api/http/response_test.go

@@ -0,0 +1,170 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+func TestError(t *testing.T) {
+	srv := NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
+
+	var resp *http.Response
+	var respbody []byte
+
+	url := srv.URL + "/this_should_fail_as_no_bzz_protocol_present"
+	resp, err := http.Get(url)
+
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+	defer resp.Body.Close()
+	respbody, err = ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI &#34;/this_should_fail_as_no_bzz_protocol_present&#34;: unknown scheme") {
+		t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
+	}
+
+	_, err = html.Parse(strings.NewReader(string(respbody)))
+	if err != nil {
+		t.Fatalf("HTML validation failed for error page returned!")
+	}
+}
+
+func Test404Page(t *testing.T) {
+	srv := NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
+
+	var resp *http.Response
+	var respbody []byte
+
+	url := srv.URL + "/bzz:/1234567890123456789012345678901234567890123456789012345678901234"
+	resp, err := http.Get(url)
+
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+	defer resp.Body.Close()
+	respbody, err = ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != 404 || !strings.Contains(string(respbody), "404") {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
+	}
+
+	_, err = html.Parse(strings.NewReader(string(respbody)))
+	if err != nil {
+		t.Fatalf("HTML validation failed for error page returned!")
+	}
+}
+
+func Test500Page(t *testing.T) {
+	srv := NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
+
+	var resp *http.Response
+	var respbody []byte
+
+	url := srv.URL + "/bzz:/thisShouldFailWith500Code"
+	resp, err := http.Get(url)
+
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+	defer resp.Body.Close()
+	respbody, err = ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != 404 {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
+	}
+
+	_, err = html.Parse(strings.NewReader(string(respbody)))
+	if err != nil {
+		t.Fatalf("HTML validation failed for error page returned!")
+	}
+}
+func Test500PageWith0xHashPrefix(t *testing.T) {
+	srv := NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
+
+	var resp *http.Response
+	var respbody []byte
+
+	url := srv.URL + "/bzz:/0xthisShouldFailWith500CodeAndAHelpfulMessage"
+	resp, err := http.Get(url)
+
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+	defer resp.Body.Close()
+	respbody, err = ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != 404 {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
+	}
+
+	if !strings.Contains(string(respbody), "The requested hash seems to be prefixed with") {
+		t.Fatalf("Did not receive the expected error message")
+	}
+
+	_, err = html.Parse(strings.NewReader(string(respbody)))
+	if err != nil {
+		t.Fatalf("HTML validation failed for error page returned!")
+	}
+}
+
+func TestJsonResponse(t *testing.T) {
+	srv := NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
+
+	var resp *http.Response
+	var respbody []byte
+
+	url := srv.URL + "/bzz:/thisShouldFailWith500Code/"
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+	req.Header.Set("Accept", "application/json")
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("Request failed: %v", err)
+	}
+
+	defer resp.Body.Close()
+	respbody, err = ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != 404 {
+		t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
+	}
+
+	if !isJSON(string(respbody)) {
+		t.Fatalf("Expected response to be JSON, received invalid JSON: %s", string(respbody))
+	}
+
+}
+
+func isJSON(s string) bool {
+	var js map[string]interface{}
+	return json.Unmarshal([]byte(s), &js) == nil
+}

api/http/roundtripper.go

@@ -0,0 +1,66 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/ethersphere/swarm/log"
+)
+
+/*
+http roundtripper to register for bzz url scheme
+see https://github.com/ethereum/go-ethereum/issues/2040
+Usage:
+
+import (
+ "github.com/ethereum/go-ethereum/common/httpclient"
+ "github.com/ethersphere/swarm/api/http"
+)
+client := httpclient.New()
+// for (private) swarm proxy running locally
+client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
+
+The port you give the Roundtripper is the port the swarm proxy is listening on.
+If Host is left empty, localhost is assumed.
+
+Using a public gateway, the above few lines gives you the leanest
+bzz-scheme aware read-only http client. You really only ever need this
+if you need go-native swarm access to bzz addresses.
+*/
+
+type RoundTripper struct {
+	Host string
+	Port string
+}
+
+func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	host := self.Host
+	if len(host) == 0 {
+		host = "localhost"
+	}
+	url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path)
+	log.Info(fmt.Sprintf("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url))
+	reqProxy, err := http.NewRequest(req.Method, url, req.Body)
+	if err != nil {
+		return nil, err
+	}
+	return http.DefaultClient.Do(reqProxy)
+}

api/http/roundtripper_test.go

@@ -0,0 +1,69 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestRoundTripper(t *testing.T) {
+	serveMux := http.NewServeMux()
+	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "GET" {
+			w.Header().Set("Content-Type", "text/plain")
+			http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(r.RequestURI))
+		} else {
+			http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
+		}
+	})
+
+	srv := httptest.NewServer(serveMux)
+	defer srv.Close()
+
+	host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
+	rt := &RoundTripper{Host: host, Port: port}
+	trans := &http.Transport{}
+	trans.RegisterProtocol("bzz", rt)
+	client := &http.Client{Transport: trans}
+	resp, err := client.Get("bzz://test.com/path")
+	if err != nil {
+		t.Errorf("expected no error, got %v", err)
+		return
+	}
+
+	defer func() {
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	content, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("expected no error, got %v", err)
+		return
+	}
+	if string(content) != "/HTTP/1.1:/test.com/path" {
+		t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content))
+	}
+
+}

api/http/sctx.go

@@ -0,0 +1,34 @@
+package http
+
+import (
+	"context"
+
+	"github.com/ethersphere/swarm/api"
+	"github.com/ethersphere/swarm/sctx"
+)
+
+type uriKey struct{}
+
+func GetRUID(ctx context.Context) string {
+	v, ok := ctx.Value(sctx.HTTPRequestIDKey{}).(string)
+	if ok {
+		return v
+	}
+	return "xxxxxxxx"
+}
+
+func SetRUID(ctx context.Context, ruid string) context.Context {
+	return context.WithValue(ctx, sctx.HTTPRequestIDKey{}, ruid)
+}
+
+func GetURI(ctx context.Context) *api.URI {
+	v, ok := ctx.Value(uriKey{}).(*api.URI)
+	if ok {
+		return v
+	}
+	return nil
+}
+
+func SetURI(ctx context.Context, uri *api.URI) context.Context {
+	return context.WithValue(ctx, uriKey{}, uri)
+}

api/http/server.go

@@ -0,0 +1,937 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+A simple http server interface to Swarm
+*/
+package http
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethersphere/swarm/api"
+	"github.com/ethersphere/swarm/chunk"
+	"github.com/ethersphere/swarm/log"
+	"github.com/ethersphere/swarm/sctx"
+	"github.com/ethersphere/swarm/storage"
+	"github.com/ethersphere/swarm/storage/feed"
+	"github.com/rs/cors"
+)
+
+var (
+	postRawCount    = metrics.NewRegisteredCounter("api.http.post.raw.count", nil)
+	postRawFail     = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil)
+	postFilesCount  = metrics.NewRegisteredCounter("api.http.post.files.count", nil)
+	postFilesFail   = metrics.NewRegisteredCounter("api.http.post.files.fail", nil)
+	deleteCount     = metrics.NewRegisteredCounter("api.http.delete.count", nil)
+	deleteFail      = metrics.NewRegisteredCounter("api.http.delete.fail", nil)
+	getCount        = metrics.NewRegisteredCounter("api.http.get.count", nil)
+	getFail         = metrics.NewRegisteredCounter("api.http.get.fail", nil)
+	getFileCount    = metrics.NewRegisteredCounter("api.http.get.file.count", nil)
+	getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil)
+	getFileFail     = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
+	getListCount    = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
+	getListFail     = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
+)
+
+const SwarmTagHeaderName = "x-swarm-tag"
+
+type methodHandler map[string]http.Handler
+
+func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	v, ok := m[r.Method]
+	if ok {
+		v.ServeHTTP(rw, r)
+		return
+	}
+	rw.WriteHeader(http.StatusMethodNotAllowed)
+}
+
+func NewServer(api *api.API, corsString string) *Server {
+	var allowedOrigins []string
+	for _, domain := range strings.Split(corsString, ",") {
+		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
+	}
+	c := cors.New(cors.Options{
+		AllowedOrigins: allowedOrigins,
+		AllowedMethods: []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodPatch, http.MethodPut},
+		MaxAge:         600,
+		AllowedHeaders: []string{"*"},
+	})
+
+	server := &Server{api: api}
+
+	defaultMiddlewares := []Adapter{
+		RecoverPanic,
+		SetRequestID,
+		SetRequestHost,
+		InitLoggingResponseWriter,
+		ParseURI,
+		InstrumentOpenTracing,
+	}
+
+	tagAdapter := Adapter(func(h http.Handler) http.Handler {
+		return InitUploadTag(h, api.Tags)
+	})
+
+	defaultPostMiddlewares := append(defaultMiddlewares, tagAdapter)
+
+	mux := http.NewServeMux()
+	mux.Handle("/bzz:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleBzzGet),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostFiles),
+			defaultPostMiddlewares...,
+		),
+		"DELETE": Adapt(
+			http.HandlerFunc(server.HandleDelete),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-raw:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostRaw),
+			defaultPostMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-immutable:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleBzzGet),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-hash:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-list:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGetList),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-feed:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGetFeed),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostFeed),
+			defaultMiddlewares...,
+		),
+	})
+
+	mux.Handle("/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleRootPaths),
+			SetRequestID,
+			InitLoggingResponseWriter,
+		),
+	})
+	server.Handler = c.Handler(mux)
+
+	return server
+}
+
+func (s *Server) ListenAndServe(addr string) error {
+	s.listenAddr = addr
+	return http.ListenAndServe(addr, s)
+}
+
+// browser API for registering bzz url scheme handlers:
+// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
+// electron (chromium) api for registering bzz url scheme handlers:
+// https://github.com/atom/electron/blob/master/docs/api/protocol.md
+type Server struct {
+	http.Handler
+	api        *api.API
+	listenAddr string
+}
+
+func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
+	log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI)
+	if r.Header.Get("Accept") == "application/x-tar" {
+		uri := GetURI(r.Context())
+		_, credentials, _ := r.BasicAuth()
+		reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri)
+		if err != nil {
+			if isDecryptError(err) {
+				w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String()))
+				respondError(w, r, err.Error(), http.StatusUnauthorized)
+				return
+			}
+			respondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
+			return
+		}
+		defer reader.Close()
+
+		w.Header().Set("Content-Type", "application/x-tar")
+
+		fileName := uri.Addr
+		if found := path.Base(uri.Path); found != "" && found != "." && found != "/" {
+			fileName = found
+		}
+		w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s.tar\"", fileName))
+
+		w.WriteHeader(http.StatusOK)
+		io.Copy(w, reader)
+		return
+	}
+
+	s.HandleGetFile(w, r)
+}
+
+func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) {
+	switch r.RequestURI {
+	case "/":
+		respondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200)
+		return
+	case "/robots.txt":
+		w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+		fmt.Fprintf(w, "User-agent: *\nDisallow: /")
+	case "/favicon.ico":
+		w.WriteHeader(http.StatusOK)
+		w.Write(faviconBytes)
+	default:
+		respondError(w, r, "Not Found", http.StatusNotFound)
+	}
+}
+
+// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
+// body in swarm and returns the resulting storage address as a text/plain response
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.post.raw", "ruid", ruid)
+
+	tagUid := sctx.GetTag(r.Context())
+	tag, err := s.api.Tags.Get(tagUid)
+	if err != nil {
+		log.Error("handle post raw got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err)
+	}
+
+	postRawCount.Inc(1)
+
+	toEncrypt := false
+	uri := GetURI(r.Context())
+	if uri.Addr == "encrypt" {
+		toEncrypt = true
+	}
+
+	if uri.Path != "" {
+		postRawFail.Inc(1)
+		respondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
+		return
+	}
+
+	if uri.Addr != "" && uri.Addr != "encrypt" {
+		postRawFail.Inc(1)
+		respondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
+		return
+	}
+
+	if r.Header.Get("Content-Length") == "" {
+		postRawFail.Inc(1)
+		respondError(w, r, "missing Content-Length header in request", http.StatusBadRequest)
+		return
+	}
+
+	addr, wait, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
+	if err != nil {
+		postRawFail.Inc(1)
+		respondError(w, r, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	wait(r.Context())
+	tag.DoneSplit(addr)
+
+	log.Debug("stored content", "ruid", ruid, "key", addr)
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, addr)
+}
+
+// HandlePostFiles handles a POST request to
+// bzz:/<hash>/<path> which contains either a single file or multiple files
+// (either a tar archive or multipart form), adds those files either to an
+// existing manifest or to a new manifest under <path> and returns the
+// resulting manifest hash as a text/plain response
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.post.files", "ruid", ruid)
+	postFilesCount.Inc(1)
+
+	contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+	if err != nil {
+		postFilesFail.Inc(1)
+		respondError(w, r, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	toEncrypt := false
+	uri := GetURI(r.Context())
+	if uri.Addr == "encrypt" {
+		toEncrypt = true
+	}
+
+	var addr storage.Address
+	if uri.Addr != "" && uri.Addr != "encrypt" {
+		addr, err = s.api.Resolve(r.Context(), uri.Addr)
+		if err != nil {
+			postFilesFail.Inc(1)
+			respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
+			return
+		}
+		log.Debug("resolved key", "ruid", ruid, "key", addr)
+	} else {
+		addr, err = s.api.NewManifest(r.Context(), toEncrypt)
+		if err != nil {
+			postFilesFail.Inc(1)
+			respondError(w, r, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		log.Debug("new manifest", "ruid", ruid, "key", addr)
+	}
+	newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
+		switch contentType {
+		case "application/x-tar":
+			_, err := s.handleTarUpload(r, mw)
+			if err != nil {
+				respondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
+				return err
+			}
+			return nil
+		case "multipart/form-data":
+			return s.handleMultipartUpload(r, params["boundary"], mw)
+
+		default:
+			return s.handleDirectUpload(r, mw)
+		}
+	})
+	if err != nil {
+		postFilesFail.Inc(1)
+		respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
+		return
+	}
+
+	tagUid := sctx.GetTag(r.Context())
+	tag, err := s.api.Tags.Get(tagUid)
+	if err != nil {
+		log.Error("got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err)
+	}
+
+	log.Debug("done splitting, setting tag total", "SPLIT", tag.Get(chunk.StateSplit), "TOTAL", tag.Total())
+	tag.DoneSplit(newAddr)
+
+	log.Debug("stored content", "ruid", ruid, "key", newAddr)
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, newAddr)
+}
+
+func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) {
+	log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()), "tag", sctx.GetTag(r.Context()))
+
+	defaultPath := r.URL.Query().Get("defaultpath")
+
+	key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, defaultPath, mw)
+	if err != nil {
+		return nil, err
+	}
+	return key, nil
+}
+
+func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api.ManifestWriter) error {
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.multipart.upload", "ruid", ruid)
+	mr := multipart.NewReader(r.Body, boundary)
+	for {
+		part, err := mr.NextPart()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return fmt.Errorf("error reading multipart form: %s", err)
+		}
+
+		var size int64
+		var reader io.Reader
+		if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
+			size, err = strconv.ParseInt(contentLength, 10, 64)
+			if err != nil {
+				return fmt.Errorf("error parsing multipart content length: %s", err)
+			}
+			reader = part
+		} else {
+			// copy the part to a tmp file to get its size
+			tmp, err := ioutil.TempFile("", "swarm-multipart")
+			if err != nil {
+				return err
+			}
+			defer os.Remove(tmp.Name())
+			defer tmp.Close()
+			size, err = io.Copy(tmp, part)
+			if err != nil {
+				return fmt.Errorf("error copying multipart content: %s", err)
+			}
+			if _, err := tmp.Seek(0, io.SeekStart); err != nil {
+				return fmt.Errorf("error copying multipart content: %s", err)
+			}
+			reader = tmp
+		}
+
+		// add the entry under the path from the request
+		name := part.FileName()
+		if name == "" {
+			name = part.FormName()
+		}
+		uri := GetURI(r.Context())
+		path := path.Join(uri.Path, name)
+		entry := &api.ManifestEntry{
+			Path:        path,
+			ContentType: part.Header.Get("Content-Type"),
+			Size:        size,
+		}
+		log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path)
+		contentKey, err := mw.AddEntry(r.Context(), reader, entry)
+		if err != nil {
+			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
+		}
+		log.Debug("stored content", "ruid", ruid, "key", contentKey)
+	}
+}
+
+func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) error {
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.direct.upload", "ruid", ruid)
+	key, err := mw.AddEntry(r.Context(), r.Body, &api.ManifestEntry{
+		Path:        GetURI(r.Context()).Path,
+		ContentType: r.Header.Get("Content-Type"),
+		Mode:        0644,
+		Size:        r.ContentLength,
+	})
+	if err != nil {
+		return err
+	}
+	log.Debug("stored content", "ruid", ruid, "key", key)
+	return nil
+}
+
+// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
+// <path> from <manifest> and returns the resulting manifest hash as a
+// text/plain response
+func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.delete", "ruid", ruid)
+	deleteCount.Inc(1)
+	newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path)
+	if err != nil {
+		deleteFail.Inc(1)
+		respondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, newKey)
+}
+
+// Handles feed manifest creation and feed updates
+// The POST request admits a JSON structure as defined in the feeds package: `feed.updateRequestJSON`
+// The requests can be to a) create a feed manifest, b) update a feed or c) both a+b: create a feed manifest and publish a first update
+func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.post.feed", "ruid", ruid)
+	var err error
+
+	// Creation and update must send feed.updateRequestJSON JSON structure
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		respondError(w, r, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query())
+	if err != nil { // couldn't parse query string or retrieve manifest
+		getFail.Inc(1)
+		httpStatus := http.StatusBadRequest
+		if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI {
+			httpStatus = http.StatusNotFound
+		}
+		respondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus)
+		return
+	}
+
+	var updateRequest feed.Request
+	updateRequest.Feed = *fd
+	query := r.URL.Query()
+
+	if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters
+		respondError(w, r, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	switch {
+	case updateRequest.IsUpdate():
+		// Verify that the signature is intact and that the signer is authorized
+		// to update this feed
+		// Check this early, to avoid creating a feed and then not being able to set its first update.
+		if err = updateRequest.Verify(); err != nil {
+			respondError(w, r, err.Error(), http.StatusForbidden)
+			return
+		}
+		_, err = s.api.FeedsUpdate(r.Context(), &updateRequest)
+		if err != nil {
+			respondError(w, r, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		fallthrough
+	case query.Get("manifest") == "1":
+		// we create a manifest so we can retrieve feed updates with bzz:// later
+		// this manifest has a special "feed type" manifest, and saves the
+		// feed identification used to retrieve feed updates later
+		m, err := s.api.NewFeedManifest(r.Context(), &updateRequest.Feed)
+		if err != nil {
+			respondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError)
+			return
+		}
+		// the key to the manifest will be passed back to the client
+		// the client can access the feed  directly through its Feed member
+		// the manifest key can be set as content in the resolver of the ENS name
+		outdata, err := json.Marshal(m)
+		if err != nil {
+			respondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
+			return
+		}
+		fmt.Fprint(w, string(outdata))
+
+		w.Header().Add("Content-type", "application/json")
+	default:
+		respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest)
+	}
+}
+
+// HandleGetFeed retrieves Swarm feeds updates:
+// bzz-feed://<manifest address or ENS name> - get latest feed update, given a manifest address
+// - or -
+// specify user + topic (optional), subtopic name (optional) directly, without manifest:
+// bzz-feed://?user=0x...&topic=0x...&name=subtopic name
+// topic defaults to 0x000... if not specified.
+// name defaults to empty string if not specified.
+// thus, empty name and topic refers to the user's default feed.
+//
+// Optional parameters:
+// time=xx - get the latest update before time (in epoch seconds)
+// hint.time=xx - hint the lookup algorithm looking for updates at around that time
+// hint.level=xx - hint the lookup algorithm looking for updates at around this frequency level
+// meta=1 - get feed metadata and status information instead of performing a feed query
+// NOTE: meta=1 will be deprecated in the near future
+func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.feed", "ruid", ruid)
+	var err error
+
+	fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query())
+	if err != nil { // couldn't parse query string or retrieve manifest
+		getFail.Inc(1)
+		httpStatus := http.StatusBadRequest
+		if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI {
+			httpStatus = http.StatusNotFound
+		}
+		respondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus)
+		return
+	}
+
+	// determine if the query specifies period and version or it is a metadata query
+	if r.URL.Query().Get("meta") == "1" {
+		unsignedUpdateRequest, err := s.api.FeedsNewRequest(r.Context(), fd)
+		if err != nil {
+			getFail.Inc(1)
+			respondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound)
+			return
+		}
+		rawResponse, err := unsignedUpdateRequest.MarshalJSON()
+		if err != nil {
+			respondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Add("Content-type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, string(rawResponse))
+		return
+	}
+
+	lookupParams := &feed.Query{Feed: *fd}
+	if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version
+		respondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest)
+		return
+	}
+
+	data, err := s.api.FeedsLookup(r.Context(), lookupParams)
+
+	// any error from the switch statement will end up here
+	if err != nil {
+		code, err2 := s.translateFeedError(w, r, "feed lookup fail", err)
+		respondError(w, r, err2.Error(), code)
+		return
+	}
+
+	// All ok, serve the retrieved update
+	log.Debug("Found update", "feed", fd.Hex(), "ruid", ruid)
+	w.Header().Set("Content-Type", api.MimeOctetStream)
+	http.ServeContent(w, r, "", time.Now(), bytes.NewReader(data))
+}
+
+func (s *Server) translateFeedError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) {
+	code := 0
+	defaultErr := fmt.Errorf("%s: %v", supErr, err)
+	rsrcErr, ok := err.(*feed.Error)
+	if !ok && rsrcErr != nil {
+		code = rsrcErr.Code()
+	}
+	switch code {
+	case storage.ErrInvalidValue:
+		return http.StatusBadRequest, defaultErr
+	case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn, storage.ErrInit:
+		return http.StatusNotFound, defaultErr
+	case storage.ErrUnauthorized, storage.ErrInvalidSignature:
+		return http.StatusUnauthorized, defaultErr
+	case storage.ErrDataOverflow:
+		return http.StatusRequestEntityTooLarge, defaultErr
+	}
+
+	return http.StatusInternalServerError, defaultErr
+}
+
+// HandleGet handles a GET request to
+// - bzz-raw://<key> and responds with the raw content stored at the
+//   given storage key
+// - bzz-hash://<key> and responds with the hash of the content stored
+//   at the given storage key as a text/plain response
+func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get", "ruid", ruid, "uri", uri)
+	getCount.Inc(1)
+	_, pass, _ := r.BasicAuth()
+
+	addr, err := s.api.ResolveURI(r.Context(), uri, pass)
+	if err != nil {
+		getFail.Inc(1)
+		respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
+		return
+	}
+	w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
+
+	log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
+
+	// if path is set, interpret <key> as a manifest and return the
+	// raw entry at the given path
+	etag := common.Bytes2Hex(addr)
+	noneMatchEtag := r.Header.Get("If-None-Match")
+	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
+	if noneMatchEtag != "" {
+		if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	switch {
+	case uri.Raw():
+		// check the root chunk exists by retrieving the file's size
+		reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
+		if _, err := reader.Size(r.Context(), nil); err != nil {
+			getFail.Inc(1)
+			respondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
+			return
+		}
+
+		w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
+
+		// allow the request to overwrite the content type using a query
+		// parameter
+		if typ := r.URL.Query().Get("content_type"); typ != "" {
+			w.Header().Set("Content-Type", typ)
+		}
+		http.ServeContent(w, r, "", time.Now(), reader)
+	case uri.Hash():
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, addr)
+	}
+
+}
+
+// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
+// a list of all files contained in <manifest> under <path> grouped into
+// common prefixes using "/" as a delimiter
+func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	_, credentials, _ := r.BasicAuth()
+	log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
+	getListCount.Inc(1)
+
+	// ensure the root path has a trailing slash so that relative URLs work
+	if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
+		return
+	}
+
+	addr, err := s.api.Resolve(r.Context(), uri.Addr)
+	if err != nil {
+		getListFail.Inc(1)
+		respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
+		return
+	}
+	log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
+
+	list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), addr, uri.Path)
+	if err != nil {
+		getListFail.Inc(1)
+		if isDecryptError(err) {
+			w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String()))
+			respondError(w, r, err.Error(), http.StatusUnauthorized)
+			return
+		}
+		respondError(w, r, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// if the client wants HTML (e.g. a browser) then render the list as a
+	// HTML index with relative URLs
+	if strings.Contains(r.Header.Get("Accept"), "text/html") {
+		w.Header().Set("Content-Type", "text/html")
+		err := TemplatesMap["bzz-list"].Execute(w, &htmlListData{
+			URI: &api.URI{
+				Scheme: "bzz",
+				Addr:   uri.Addr,
+				Path:   uri.Path,
+			},
+			List: &list,
+		})
+		if err != nil {
+			getListFail.Inc(1)
+			log.Error(fmt.Sprintf("error rendering list HTML: %s", err))
+		}
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(&list)
+}
+
+// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
+// with the content of the file at <path> from the given <manifest>
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	_, credentials, _ := r.BasicAuth()
+	log.Debug("handle.get.file", "ruid", ruid, "uri", r.RequestURI)
+	getFileCount.Inc(1)
+
+	// ensure the root path has a trailing slash so that relative URLs work
+	if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
+		return
+	}
+	var err error
+	manifestAddr := uri.Address()
+
+	if manifestAddr == nil {
+		manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr)
+		if err != nil {
+			getFileFail.Inc(1)
+			respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
+			return
+		}
+	} else {
+		w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
+	}
+
+	log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
+
+	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
+
+	etag := common.Bytes2Hex(contentKey)
+	noneMatchEtag := r.Header.Get("If-None-Match")
+	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key.
+	if noneMatchEtag != "" {
+		if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	if err != nil {
+		if isDecryptError(err) {
+			w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
+			respondError(w, r, err.Error(), http.StatusUnauthorized)
+			return
+		}
+
+		switch status {
+		case http.StatusNotFound:
+			getFileNotFound.Inc(1)
+			respondError(w, r, err.Error(), http.StatusNotFound)
+		default:
+			getFileFail.Inc(1)
+			respondError(w, r, err.Error(), http.StatusInternalServerError)
+		}
+		return
+	}
+
+	//the request results in ambiguous files
+	//e.g. /read with readme.md and readinglist.txt available in manifest
+	if status == http.StatusMultipleChoices {
+		list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
+		if err != nil {
+			getFileFail.Inc(1)
+			if isDecryptError(err) {
+				w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
+				respondError(w, r, err.Error(), http.StatusUnauthorized)
+				return
+			}
+			respondError(w, r, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", ruid)
+		//show a nice page links to available entries
+		ShowMultipleChoices(w, r, list)
+		return
+	}
+
+	// check the root chunk exists by retrieving the file's size
+	if _, err := reader.Size(r.Context(), nil); err != nil {
+		getFileNotFound.Inc(1)
+		respondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound)
+		return
+	}
+
+	if contentType != "" {
+		w.Header().Set("Content-Type", contentType)
+	}
+
+	fileName := uri.Addr
+	if found := path.Base(uri.Path); found != "" && found != "." && found != "/" {
+		fileName = found
+	}
+	w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", fileName))
+
+	http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
+}
+
+// calculateNumberOfChunks calculates the number of chunks in an arbitrary content length
+func calculateNumberOfChunks(contentLength int64, isEncrypted bool) int64 {
+	if contentLength < 4096 {
+		return 1
+	}
+	branchingFactor := 128
+	if isEncrypted {
+		branchingFactor = 64
+	}
+
+	dataChunks := math.Ceil(float64(contentLength) / float64(4096))
+	totalChunks := dataChunks
+	intermediate := dataChunks / float64(branchingFactor)
+
+	for intermediate > 1 {
+		totalChunks += math.Ceil(intermediate)
+		intermediate = intermediate / float64(branchingFactor)
+	}
+
+	return int64(totalChunks) + 1
+}
+
+// The size of buffer used for bufio.Reader on LazyChunkReader passed to
+// http.ServeContent in HandleGetFile.
+// Warning: This value influences the number of chunk requests and chunker join goroutines
+// per file request.
+// Recommended value is 4 times the io.Copy default buffer value which is 32kB.
+const getFileBufferSize = 4 * 32 * 1024
+
+// bufferedReadSeeker wraps bufio.Reader to expose Seek method
+// from the provied io.ReadSeeker in newBufferedReadSeeker.
+type bufferedReadSeeker struct {
+	r io.Reader
+	s io.Seeker
+}
+
+// newBufferedReadSeeker creates a new instance of bufferedReadSeeker,
+// out of io.ReadSeeker. Argument `size` is the size of the read buffer.
+func newBufferedReadSeeker(readSeeker io.ReadSeeker, size int) bufferedReadSeeker {
+	return bufferedReadSeeker{
+		r: bufio.NewReaderSize(readSeeker, size),
+		s: readSeeker,
+	}
+}
+
+func (b bufferedReadSeeker) Read(p []byte) (n int, err error) {
+	return b.r.Read(p)
+}
+
+func (b bufferedReadSeeker) Seek(offset int64, whence int) (int64, error) {
+	return b.s.Seek(offset, whence)
+}
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func newLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}
+
+func isDecryptError(err error) bool {
+	return strings.Contains(err.Error(), api.ErrDecrypt.Error())
+}

api/http/test_server.go

@@ -0,0 +1,100 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/ethersphere/swarm/api"
+	"github.com/ethersphere/swarm/chunk"
+	"github.com/ethersphere/swarm/storage"
+	"github.com/ethersphere/swarm/storage/feed"
+	"github.com/ethersphere/swarm/storage/localstore"
+)
+
+type TestServer interface {
+	ServeHTTP(http.ResponseWriter, *http.Request)
+}
+
+func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, resolver api.Resolver) *TestSwarmServer {
+	swarmDir, err := ioutil.TempDir("", "swarm-storage-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	localStore, err := localstore.New(swarmDir, make([]byte, 32), nil)
+	if err != nil {
+		os.RemoveAll(swarmDir)
+		t.Fatal(err)
+	}
+
+	tags := chunk.NewTags()
+	fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams(), tags)
+
+	// Swarm feeds test setup
+	feedsDir, err := ioutil.TempDir("", "swarm-feeds-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	feeds, err := feed.NewTestHandler(feedsDir, &feed.HandlerParams{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	swarmApi := api.NewAPI(fileStore, resolver, feeds.Handler, nil, tags)
+	apiServer := httptest.NewServer(serverFunc(swarmApi))
+
+	tss := &TestSwarmServer{
+		Server:    apiServer,
+		FileStore: fileStore,
+		Tags:      tags,
+		dir:       swarmDir,
+		Hasher:    storage.MakeHashFunc(storage.DefaultHash)(),
+		cleanup: func() {
+			apiServer.Close()
+			fileStore.Close()
+			feeds.Close()
+			os.RemoveAll(swarmDir)
+			os.RemoveAll(feedsDir)
+		},
+		CurrentTime: 42,
+	}
+	feed.TimestampProvider = tss
+	return tss
+}
+
+type TestSwarmServer struct {
+	*httptest.Server
+	Hasher      storage.SwarmHash
+	FileStore   *storage.FileStore
+	Tags        *chunk.Tags
+	dir         string
+	cleanup     func()
+	CurrentTime uint64
+}
+
+func (t *TestSwarmServer) Close() {
+	t.cleanup()
+}
+
+func (t *TestSwarmServer) Now() feed.Timestamp {
+	return feed.Timestamp{Time: t.CurrentTime}
+}