swarm: mock store listings (#19157)

* swarm/storage/mock: implement listings methods for mem and rpc stores

* swarm/storage/mock/rpc: add comments and newTestStore helper function

* swarm/storage/mock/mem: add missing comments

* swarm/storage/mock: add comments to new types and constants

* swarm/storage/mock/db: implement listings for mock/db global store

* swarm/storage/mock/test: add comments for MockStoreListings

* swarm/storage/mock/explorer: initial implementation

* cmd/swarm/global-store: add chunk explorer

* cmd/swarm/global-store: add chunk explorer tests

* swarm/storage/mock/explorer: add tests

* swarm/storage/mock/explorer: add swagger api definition

* swarm/storage/mock/explorer: not-zero test values for invalid addr and key

* swarm/storage/mock/explorer: test wildcard cors origin

* swarm/storage/mock/db: renames based on Fabio's suggestions

* swarm/storage/mock/explorer: add more comments to testHandler function

* cmd/swarm/global-store: terminate subprocess with Kill in tests

cmd/swarm/global-store/explorer.go

@@ -0,0 +1,66 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/explorer"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+// serveChunkExplorer starts an http server in background with chunk explorer handler
+// using the provided global store. Server is started if the returned shutdown function
+// is not nil.
+func serveChunkExplorer(ctx *cli.Context, globalStore mock.GlobalStorer) (shutdown func(), err error) {
+	if !ctx.IsSet("explorer-address") {
+		return nil, nil
+	}
+
+	corsOrigins := ctx.StringSlice("explorer-cors-origin")
+	server := &http.Server{
+		Handler:      explorer.NewHandler(globalStore, corsOrigins),
+		IdleTimeout:  30 * time.Minute,
+		ReadTimeout:  2 * time.Minute,
+		WriteTimeout: 2 * time.Minute,
+	}
+	listener, err := net.Listen("tcp", ctx.String("explorer-address"))
+	if err != nil {
+		return nil, fmt.Errorf("explorer: %v", err)
+	}
+	log.Info("chunk explorer http", "address", listener.Addr().String(), "origins", corsOrigins)
+
+	go func() {
+		if err := server.Serve(listener); err != nil {
+			log.Error("chunk explorer", "err", err)
+		}
+	}()
+
+	return func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := server.Shutdown(ctx); err != nil {
+			log.Error("chunk explorer: shutdown", "err", err)
+		}
+	}, nil
+}

cmd/swarm/global-store/explorer_test.go

@@ -0,0 +1,254 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/explorer"
+	mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
+)
+
+// TestExplorer validates basic chunk explorer functionality by storing
+// a small set of chunk and making http requests on exposed endpoint.
+// Full chunk explorer validation is done in mock/explorer package.
+func TestExplorer(t *testing.T) {
+	addr := findFreeTCPAddress(t)
+	explorerAddr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, "ws", "--addr", addr, "--explorer-address", explorerAddr)
+	defer testCmd.Kill()
+
+	client := websocketClient(t, addr)
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	nodeKeys := map[string][]string{
+		"a1": {"b1", "b2", "b3"},
+		"a2": {"b3", "b4", "b5"},
+	}
+
+	keyNodes := make(map[string][]string)
+
+	for addr, keys := range nodeKeys {
+		for _, key := range keys {
+			keyNodes[key] = append(keyNodes[key], addr)
+		}
+	}
+
+	invalidAddr := "c1"
+	invalidKey := "d1"
+
+	for addr, keys := range nodeKeys {
+		for _, key := range keys {
+			err := store.Put(common.HexToAddress(addr), common.Hex2Bytes(key), []byte("data"))
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	endpoint := "http://" + explorerAddr
+
+	t.Run("has key", func(t *testing.T) {
+		for addr, keys := range nodeKeys {
+			for _, key := range keys {
+				testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+key, http.StatusOK)
+				testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+key, http.StatusNotFound)
+			}
+			testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+invalidKey, http.StatusNotFound)
+		}
+		testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+invalidKey, http.StatusNotFound)
+	})
+
+	t.Run("keys", func(t *testing.T) {
+		var keys []string
+		for key := range keyNodes {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		testKeysResponse(t, endpoint+"/api/keys", explorer.KeysResponse{
+			Keys: keys,
+		})
+	})
+
+	t.Run("nodes", func(t *testing.T) {
+		var nodes []string
+		for addr := range nodeKeys {
+			nodes = append(nodes, common.HexToAddress(addr).Hex())
+		}
+		sort.Strings(nodes)
+		testNodesResponse(t, endpoint+"/api/nodes", explorer.NodesResponse{
+			Nodes: nodes,
+		})
+	})
+
+	t.Run("node keys", func(t *testing.T) {
+		for addr, keys := range nodeKeys {
+			testKeysResponse(t, endpoint+"/api/keys?node="+addr, explorer.KeysResponse{
+				Keys: keys,
+			})
+		}
+		testKeysResponse(t, endpoint+"/api/keys?node="+invalidAddr, explorer.KeysResponse{})
+	})
+
+	t.Run("key nodes", func(t *testing.T) {
+		for key, addrs := range keyNodes {
+			var nodes []string
+			for _, addr := range addrs {
+				nodes = append(nodes, common.HexToAddress(addr).Hex())
+			}
+			sort.Strings(nodes)
+			testNodesResponse(t, endpoint+"/api/nodes?key="+key, explorer.NodesResponse{
+				Nodes: nodes,
+			})
+		}
+		testNodesResponse(t, endpoint+"/api/nodes?key="+invalidKey, explorer.NodesResponse{})
+	})
+}
+
+// TestExplorer_CORSOrigin validates if chunk explorer returns
+// correct CORS origin header in GET and OPTIONS requests.
+func TestExplorer_CORSOrigin(t *testing.T) {
+	origin := "http://localhost/"
+	addr := findFreeTCPAddress(t)
+	explorerAddr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, "ws",
+		"--addr", addr,
+		"--explorer-address", explorerAddr,
+		"--explorer-cors-origin", origin,
+	)
+	defer testCmd.Kill()
+
+	// wait until the server is started
+	waitHTTPEndpoint(t, explorerAddr)
+
+	url := "http://" + explorerAddr + "/api/keys"
+
+	t.Run("get", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Set("Origin", origin)
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		header := resp.Header.Get("Access-Control-Allow-Origin")
+		if header != origin {
+			t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin)
+		}
+	})
+
+	t.Run("preflight", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodOptions, url, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		req.Header.Set("Origin", origin)
+		req.Header.Set("Access-Control-Request-Method", "GET")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		header := resp.Header.Get("Access-Control-Allow-Origin")
+		if header != origin {
+			t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin)
+		}
+	})
+}
+
+// testStatusResponse makes an http request to provided url
+// and validates if response is explorer.StatusResponse for
+// the expected status code.
+func testStatusResponse(t *testing.T, url string, code int) {
+	t.Helper()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != code {
+		t.Errorf("got status code %v, want %v", resp.StatusCode, code)
+	}
+	var r explorer.StatusResponse
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		t.Fatal(err)
+	}
+	if r.Code != code {
+		t.Errorf("got response code %v, want %v", r.Code, code)
+	}
+	if r.Message != http.StatusText(code) {
+		t.Errorf("got response message %q, want %q", r.Message, http.StatusText(code))
+	}
+}
+
+// testKeysResponse makes an http request to provided url
+// and validates if response machhes expected explorer.KeysResponse.
+func testKeysResponse(t *testing.T, url string, want explorer.KeysResponse) {
+	t.Helper()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK)
+	}
+	var r explorer.KeysResponse
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprint(r.Keys) != fmt.Sprint(want.Keys) {
+		t.Errorf("got keys %v, want %v", r.Keys, want.Keys)
+	}
+	if r.Next != want.Next {
+		t.Errorf("got next %s, want %s", r.Next, want.Next)
+	}
+}
+
+// testNodeResponse makes an http request to provided url
+// and validates if response machhes expected explorer.NodeResponse.
+func testNodesResponse(t *testing.T, url string, want explorer.NodesResponse) {
+	t.Helper()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK)
+	}
+	var r explorer.NodesResponse
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		t.Fatal(err)
+	}
+	if fmt.Sprint(r.Nodes) != fmt.Sprint(want.Nodes) {
+		t.Errorf("got nodes %v, want %v", r.Nodes, want.Nodes)
+	}
+	if r.Next != want.Next {
+		t.Errorf("got next %s, want %s", r.Next, want.Next)
+	}
+}

cmd/swarm/global-store/global_store.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -66,7 +67,7 @@ func startWS(ctx *cli.Context) (err error) {
 	return http.Serve(listener, server.WebsocketHandler(origins))
 }
 
-// newServer creates a global store and returns its RPC server.
+// newServer creates a global store and starts a chunk explorer server if configured.
 // Returned cleanup function should be called only if err is nil.
 func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
 	log.PrintOrigins(true)
@@ -81,7 +82,9 @@ func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error)
 			return nil, nil, err
 		}
 		cleanup = func() {
-			dbStore.Close()
+			if err := dbStore.Close(); err != nil {
+				log.Error("global store: close", "err", err)
+			}
 		}
 		globalStore = dbStore
 		log.Info("database global store", "dir", dir)
@@ -96,5 +99,22 @@ func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error)
 		return nil, nil, err
 	}
 
+	shutdown, err := serveChunkExplorer(ctx, globalStore)
+	if err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+	if shutdown != nil {
+		cleanup = func() {
+			shutdown()
+
+			if c, ok := globalStore.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Error("global store: close", "err", err)
+				}
+			}
+		}
+	}
+
 	return server, cleanup, nil
 }

cmd/swarm/global-store/global_store_test.go

@@ -69,16 +69,7 @@ func testHTTP(t *testing.T, put bool, args ...string) {
 
 	// wait until global store process is started as
 	// rpc.DialHTTP is actually not connecting
-	for i := 0; i < 1000; i++ {
-		_, err = http.DefaultClient.Get("http://" + addr)
-		if err == nil {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
+	waitHTTPEndpoint(t, addr)
 
 	store := mockRPC.NewGlobalStore(client)
 	defer store.Close()
@@ -137,19 +128,7 @@ func testWebsocket(t *testing.T, put bool, args ...string) {
 	testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
 	defer testCmd.Kill()
 
-	var client *rpc.Client
-	var err error
-	// wait until global store process is started
-	for i := 0; i < 1000; i++ {
-		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
-		if err == nil {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
+	client := websocketClient(t, addr)
 
 	store := mockRPC.NewGlobalStore(client)
 	defer store.Close()
@@ -160,7 +139,7 @@ func testWebsocket(t *testing.T, put bool, args ...string) {
 	wantValue := "value"
 
 	if put {
-		err = node.Put([]byte(wantKey), []byte(wantValue))
+		err := node.Put([]byte(wantKey), []byte(wantValue))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -189,3 +168,40 @@ func findFreeTCPAddress(t *testing.T) (addr string) {
 
 	return listener.Addr().String()
 }
+
+// websocketClient waits until global store process is started
+// and returns rpc client.
+func websocketClient(t *testing.T, addr string) (client *rpc.Client) {
+	t.Helper()
+
+	var err error
+	for i := 0; i < 1000; i++ {
+		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return client
+}
+
+// waitHTTPEndpoint retries http requests to a provided
+// address until the connection is established.
+func waitHTTPEndpoint(t *testing.T, addr string) {
+	t.Helper()
+
+	var err error
+	for i := 0; i < 1000; i++ {
+		_, err = http.Get("http://" + addr)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+}

cmd/swarm/global-store/main.go

@@ -19,12 +19,14 @@ package main
 import (
 	"os"
 
-	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/log"
 	cli "gopkg.in/urfave/cli.v1"
 )
 
-var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+var (
+	version   = "0.1"
+	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+)
 
 func main() {
 	err := newApp().Run(os.Args)
@@ -37,16 +39,30 @@ func main() {
 // newApp construct a new instance of Swarm Global Store.
 // Method Run is called on it in the main function and in tests.
 func newApp() (app *cli.App) {
-	app = utils.NewApp(gitCommit, "Swarm Global Store")
-
+	app = cli.NewApp()
 	app.Name = "global-store"
+	app.Version = version
+	if len(gitCommit) >= 8 {
+		app.Version += "-" + gitCommit[:8]
+	}
+	app.Usage = "Swarm Global Store"
 
 	// app flags (for all commands)
 	app.Flags = []cli.Flag{
 		cli.IntFlag{
 			Name:  "verbosity",
 			Value: 3,
-			Usage: "verbosity level",
+			Usage: "Verbosity level.",
+		},
+		cli.StringFlag{
+			Name:  "explorer-address",
+			Value: "",
+			Usage: "Chunk explorer HTTP listener address.",
+		},
+		cli.StringSliceFlag{
+			Name:  "explorer-cors-origin",
+			Value: nil,
+			Usage: "Chunk explorer CORS origin (can be specified multiple times).",
 		},
 	}
 
@@ -54,7 +70,7 @@ func newApp() (app *cli.App) {
 		{
 			Name:    "http",
 			Aliases: []string{"h"},
-			Usage:   "start swarm global store with http server",
+			Usage:   "Start swarm global store with HTTP server.",
 			Action:  startHTTP,
 			// Flags only for "start" command.
 			// Allow app flags to be specified after the
@@ -63,19 +79,19 @@ func newApp() (app *cli.App) {
 				cli.StringFlag{
 					Name:  "dir",
 					Value: "",
-					Usage: "data directory",
+					Usage: "Data directory.",
 				},
 				cli.StringFlag{
 					Name:  "addr",
 					Value: "0.0.0.0:3033",
-					Usage: "address to listen for http connection",
+					Usage: "Address to listen for HTTP connections.",
 				},
 			),
 		},
 		{
 			Name:    "websocket",
 			Aliases: []string{"ws"},
-			Usage:   "start swarm global store with websocket server",
+			Usage:   "Start swarm global store with WebSocket server.",
 			Action:  startWS,
 			// Flags only for "start" command.
 			// Allow app flags to be specified after the
@@ -84,17 +100,17 @@ func newApp() (app *cli.App) {
 				cli.StringFlag{
 					Name:  "dir",
 					Value: "",
-					Usage: "data directory",
+					Usage: "Data directory.",
 				},
 				cli.StringFlag{
 					Name:  "addr",
 					Value: "0.0.0.0:3033",
-					Usage: "address to listen for websocket connection",
+					Usage: "Address to listen for WebSocket connections.",
 				},
 				cli.StringSliceFlag{
-					Name:  "origins",
-					Value: &cli.StringSlice{"*"},
-					Usage: "websocket origins",
+					Name:  "origin",
+					Value: nil,
+					Usage: "WebSocket CORS origin (can be specified multiple times).",
 				},
 			),
 		},