rpc: implement full bi-directional communication (#18471)

New APIs added:

    client.RegisterName(namespace, service) // makes service available to server
    client.Notify(ctx, method, args...)     // sends a notification
    ClientFromContext(ctx)                  // to get a client in handler method

This is essentially a rewrite of the server-side code. JSON-RPC
processing code is now the same on both server and client side. Many
minor issues were fixed in the process and there is a new test suite for
JSON-RPC spec compliance (and non-compliance in some cases).

List of behavior changes:

- Method handlers are now called with a per-request context instead of a
  per-connection context. The context is canceled right after the method
  returns.
- Subscription error channels are always closed when the connection
  ends. There is no need to also wait on the Notifier's Closed channel
  to detect whether the subscription has ended.
- Client now omits "params" instead of sending "params": null when there
  are no arguments to a call. The previous behavior was not compliant
  with the spec. The server still accepts "params": null.
- Floating point numbers are allowed as "id". The spec doesn't allow
  them, but we handle request "id" as json.RawMessage and guarantee that
  the same number will be sent back.
- Logging is improved significantly. There is now a message at DEBUG
  level for each RPC call served.
This commit is contained in:
Felix Lange
2019-02-04 13:47:34 +01:00
committed by GitHub
parent ec3432bccb
commit 245f3146c2
36 changed files with 2211 additions and 2169 deletions

View File

@ -18,36 +18,104 @@ package rpc
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"strconv"
"strings"
"sync"
"github.com/ethereum/go-ethereum/log"
"time"
)
const (
jsonrpcVersion = "2.0"
vsn = "2.0"
serviceMethodSeparator = "_"
subscribeMethodSuffix = "_subscribe"
unsubscribeMethodSuffix = "_unsubscribe"
notificationMethodSuffix = "_subscription"
defaultWriteTimeout = 10 * time.Second // used if context has no deadline
)
type jsonRequest struct {
Method string `json:"method"`
Version string `json:"jsonrpc"`
Id json.RawMessage `json:"id,omitempty"`
Payload json.RawMessage `json:"params,omitempty"`
var null = json.RawMessage("null")
type subscriptionResult struct {
ID string `json:"subscription"`
Result json.RawMessage `json:"result,omitempty"`
}
type jsonSuccessResponse struct {
Version string `json:"jsonrpc"`
Id interface{} `json:"id,omitempty"`
Result interface{} `json:"result"`
// A value of this type can a JSON-RPC request, notification, successful response or
// error response. Which one it is depends on the fields.
type jsonrpcMessage struct {
Version string `json:"jsonrpc,omitempty"`
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Error *jsonError `json:"error,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
}
func (msg *jsonrpcMessage) isNotification() bool {
return msg.ID == nil && msg.Method != ""
}
func (msg *jsonrpcMessage) isCall() bool {
return msg.hasValidID() && msg.Method != ""
}
func (msg *jsonrpcMessage) isResponse() bool {
return msg.hasValidID() && msg.Method == "" && msg.Params == nil && (msg.Result != nil || msg.Error != nil)
}
func (msg *jsonrpcMessage) hasValidID() bool {
return len(msg.ID) > 0 && msg.ID[0] != '{' && msg.ID[0] != '['
}
func (msg *jsonrpcMessage) isSubscribe() bool {
return strings.HasSuffix(msg.Method, subscribeMethodSuffix)
}
func (msg *jsonrpcMessage) isUnsubscribe() bool {
return strings.HasSuffix(msg.Method, unsubscribeMethodSuffix)
}
func (msg *jsonrpcMessage) namespace() string {
elem := strings.SplitN(msg.Method, serviceMethodSeparator, 2)
return elem[0]
}
func (msg *jsonrpcMessage) String() string {
b, _ := json.Marshal(msg)
return string(b)
}
func (msg *jsonrpcMessage) errorResponse(err error) *jsonrpcMessage {
resp := errorMessage(err)
resp.ID = msg.ID
return resp
}
func (msg *jsonrpcMessage) response(result interface{}) *jsonrpcMessage {
enc, err := json.Marshal(result)
if err != nil {
// TODO: wrap with 'internal server error'
return msg.errorResponse(err)
}
return &jsonrpcMessage{Version: vsn, ID: msg.ID, Result: enc}
}
func errorMessage(err error) *jsonrpcMessage {
msg := &jsonrpcMessage{Version: vsn, ID: null, Error: &jsonError{
Code: defaultErrorCode,
Message: err.Error(),
}}
ec, ok := err.(Error)
if ok {
msg.Error.Code = ec.ErrorCode()
}
return msg
}
type jsonError struct {
@ -56,35 +124,6 @@ type jsonError struct {
Data interface{} `json:"data,omitempty"`
}
type jsonErrResponse struct {
Version string `json:"jsonrpc"`
Id interface{} `json:"id,omitempty"`
Error jsonError `json:"error"`
}
type jsonSubscription struct {
Subscription string `json:"subscription"`
Result interface{} `json:"result,omitempty"`
}
type jsonNotification struct {
Version string `json:"jsonrpc"`
Method string `json:"method"`
Params jsonSubscription `json:"params"`
}
// jsonCodec reads and writes JSON-RPC messages to the underlying connection. It
// also has support for parsing arguments and serializing (result) objects.
type jsonCodec struct {
closer sync.Once // close closed channel once
closed chan interface{} // closed on Close
decMu sync.Mutex // guards the decoder
decode func(v interface{}) error // decoder to allow multiple transports
encMu sync.Mutex // guards the encoder
encode func(v interface{}) error // encoder to allow multiple transports
rw io.ReadWriteCloser // connection
}
func (err *jsonError) Error() string {
if err.Message == "" {
return fmt.Sprintf("json-rpc error %d", err.Code)
@ -96,34 +135,126 @@ func (err *jsonError) ErrorCode() int {
return err.Code
}
// Conn is a subset of the methods of net.Conn which are sufficient for ServerCodec.
type Conn interface {
io.ReadWriteCloser
SetWriteDeadline(time.Time) error
}
// ConnRemoteAddr wraps the RemoteAddr operation, which returns a description
// of the peer address of a connection. If a Conn also implements ConnRemoteAddr, this
// description is used in log messages.
type ConnRemoteAddr interface {
RemoteAddr() string
}
// connWithRemoteAddr overrides the remote address of a connection.
type connWithRemoteAddr struct {
Conn
addr string
}
func (c connWithRemoteAddr) RemoteAddr() string { return c.addr }
// jsonCodec reads and writes JSON-RPC messages to the underlying connection. It also has
// support for parsing arguments and serializing (result) objects.
type jsonCodec struct {
remoteAddr string
closer sync.Once // close closed channel once
closed chan interface{} // closed on Close
decode func(v interface{}) error // decoder to allow multiple transports
encMu sync.Mutex // guards the encoder
encode func(v interface{}) error // encoder to allow multiple transports
conn Conn
}
// NewCodec creates a new RPC server codec with support for JSON-RPC 2.0 based
// on explicitly given encoding and decoding methods.
func NewCodec(rwc io.ReadWriteCloser, encode, decode func(v interface{}) error) ServerCodec {
return &jsonCodec{
func NewCodec(conn Conn, encode, decode func(v interface{}) error) ServerCodec {
codec := &jsonCodec{
closed: make(chan interface{}),
encode: encode,
decode: decode,
rw: rwc,
conn: conn,
}
if ra, ok := conn.(ConnRemoteAddr); ok {
codec.remoteAddr = ra.RemoteAddr()
}
return codec
}
// NewJSONCodec creates a new RPC server codec with support for JSON-RPC 2.0.
func NewJSONCodec(rwc io.ReadWriteCloser) ServerCodec {
enc := json.NewEncoder(rwc)
dec := json.NewDecoder(rwc)
func NewJSONCodec(conn Conn) ServerCodec {
enc := json.NewEncoder(conn)
dec := json.NewDecoder(conn)
dec.UseNumber()
return NewCodec(conn, enc.Encode, dec.Decode)
}
return &jsonCodec{
closed: make(chan interface{}),
encode: enc.Encode,
decode: dec.Decode,
rw: rwc,
func (c *jsonCodec) RemoteAddr() string {
return c.remoteAddr
}
func (c *jsonCodec) Read() (msg []*jsonrpcMessage, batch bool, err error) {
// Decode the next JSON object in the input stream.
// This verifies basic syntax, etc.
var rawmsg json.RawMessage
if err := c.decode(&rawmsg); err != nil {
return nil, false, err
}
msg, batch = parseMessage(rawmsg)
return msg, batch, nil
}
// Write sends a message to client.
func (c *jsonCodec) Write(ctx context.Context, v interface{}) error {
c.encMu.Lock()
defer c.encMu.Unlock()
deadline, ok := ctx.Deadline()
if !ok {
deadline = time.Now().Add(defaultWriteTimeout)
}
c.conn.SetWriteDeadline(deadline)
return c.encode(v)
}
// Close the underlying connection
func (c *jsonCodec) Close() {
c.closer.Do(func() {
close(c.closed)
c.conn.Close()
})
}
// Closed returns a channel which will be closed when Close is called
func (c *jsonCodec) Closed() <-chan interface{} {
return c.closed
}
// parseMessage parses raw bytes as a (batch of) JSON-RPC message(s). There are no error
// checks in this function because the raw message has already been syntax-checked when it
// is called. Any non-JSON-RPC messages in the input return the zero value of
// jsonrpcMessage.
func parseMessage(raw json.RawMessage) ([]*jsonrpcMessage, bool) {
if !isBatch(raw) {
msgs := []*jsonrpcMessage{{}}
json.Unmarshal(raw, &msgs[0])
return msgs, false
}
dec := json.NewDecoder(bytes.NewReader(raw))
dec.Token() // skip '['
var msgs []*jsonrpcMessage
for dec.More() {
msgs = append(msgs, new(jsonrpcMessage))
dec.Decode(&msgs[len(msgs)-1])
}
return msgs, true
}
// isBatch returns true when the first non-whitespace characters is '['
func isBatch(msg json.RawMessage) bool {
for _, c := range msg {
func isBatch(raw json.RawMessage) bool {
for _, c := range raw {
// skip insignificant whitespace (http://www.ietf.org/rfc/rfc4627.txt)
if c == 0x20 || c == 0x09 || c == 0x0a || c == 0x0d {
continue
@ -133,231 +264,67 @@ func isBatch(msg json.RawMessage) bool {
return false
}
// ReadRequestHeaders will read new requests without parsing the arguments. It will
// return a collection of requests, an indication if these requests are in batch
// form or an error when the incoming message could not be read/parsed.
func (c *jsonCodec) ReadRequestHeaders() ([]rpcRequest, bool, Error) {
c.decMu.Lock()
defer c.decMu.Unlock()
var incomingMsg json.RawMessage
if err := c.decode(&incomingMsg); err != nil {
return nil, false, &invalidRequestError{err.Error()}
}
if isBatch(incomingMsg) {
return parseBatchRequest(incomingMsg)
}
return parseRequest(incomingMsg)
}
// checkReqId returns an error when the given reqId isn't valid for RPC method calls.
// valid id's are strings, numbers or null
func checkReqId(reqId json.RawMessage) error {
if len(reqId) == 0 {
return fmt.Errorf("missing request id")
}
if _, err := strconv.ParseFloat(string(reqId), 64); err == nil {
return nil
}
var str string
if err := json.Unmarshal(reqId, &str); err == nil {
return nil
}
return fmt.Errorf("invalid request id")
}
// parseRequest will parse a single request from the given RawMessage. It will return
// the parsed request, an indication if the request was a batch or an error when
// the request could not be parsed.
func parseRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) {
var in jsonRequest
if err := json.Unmarshal(incomingMsg, &in); err != nil {
return nil, false, &invalidMessageError{err.Error()}
}
if err := checkReqId(in.Id); err != nil {
return nil, false, &invalidMessageError{err.Error()}
}
// subscribe are special, they will always use `subscribeMethod` as first param in the payload
if strings.HasSuffix(in.Method, subscribeMethodSuffix) {
reqs := []rpcRequest{{id: &in.Id, isPubSub: true}}
if len(in.Payload) > 0 {
// first param must be subscription name
var subscribeMethod [1]string
if err := json.Unmarshal(in.Payload, &subscribeMethod); err != nil {
log.Debug(fmt.Sprintf("Unable to parse subscription method: %v\n", err))
return nil, false, &invalidRequestError{"Unable to parse subscription request"}
}
reqs[0].service, reqs[0].method = strings.TrimSuffix(in.Method, subscribeMethodSuffix), subscribeMethod[0]
reqs[0].params = in.Payload
return reqs, false, nil
}
return nil, false, &invalidRequestError{"Unable to parse subscription request"}
}
if strings.HasSuffix(in.Method, unsubscribeMethodSuffix) {
return []rpcRequest{{id: &in.Id, isPubSub: true,
method: in.Method, params: in.Payload}}, false, nil
}
elems := strings.Split(in.Method, serviceMethodSeparator)
if len(elems) != 2 {
return nil, false, &methodNotFoundError{in.Method, ""}
}
// regular RPC call
if len(in.Payload) == 0 {
return []rpcRequest{{service: elems[0], method: elems[1], id: &in.Id}}, false, nil
}
return []rpcRequest{{service: elems[0], method: elems[1], id: &in.Id, params: in.Payload}}, false, nil
}
// parseBatchRequest will parse a batch request into a collection of requests from the given RawMessage, an indication
// if the request was a batch or an error when the request could not be read.
func parseBatchRequest(incomingMsg json.RawMessage) ([]rpcRequest, bool, Error) {
var in []jsonRequest
if err := json.Unmarshal(incomingMsg, &in); err != nil {
return nil, false, &invalidMessageError{err.Error()}
}
requests := make([]rpcRequest, len(in))
for i, r := range in {
if err := checkReqId(r.Id); err != nil {
return nil, false, &invalidMessageError{err.Error()}
}
id := &in[i].Id
// subscribe are special, they will always use `subscriptionMethod` as first param in the payload
if strings.HasSuffix(r.Method, subscribeMethodSuffix) {
requests[i] = rpcRequest{id: id, isPubSub: true}
if len(r.Payload) > 0 {
// first param must be subscription name
var subscribeMethod [1]string
if err := json.Unmarshal(r.Payload, &subscribeMethod); err != nil {
log.Debug(fmt.Sprintf("Unable to parse subscription method: %v\n", err))
return nil, false, &invalidRequestError{"Unable to parse subscription request"}
}
requests[i].service, requests[i].method = strings.TrimSuffix(r.Method, subscribeMethodSuffix), subscribeMethod[0]
requests[i].params = r.Payload
continue
}
return nil, true, &invalidRequestError{"Unable to parse (un)subscribe request arguments"}
}
if strings.HasSuffix(r.Method, unsubscribeMethodSuffix) {
requests[i] = rpcRequest{id: id, isPubSub: true, method: r.Method, params: r.Payload}
continue
}
if len(r.Payload) == 0 {
requests[i] = rpcRequest{id: id, params: nil}
} else {
requests[i] = rpcRequest{id: id, params: r.Payload}
}
if elem := strings.Split(r.Method, serviceMethodSeparator); len(elem) == 2 {
requests[i].service, requests[i].method = elem[0], elem[1]
} else {
requests[i].err = &methodNotFoundError{r.Method, ""}
}
}
return requests, true, nil
}
// ParseRequestArguments tries to parse the given params (json.RawMessage) with the given
// types. It returns the parsed values or an error when the parsing failed.
func (c *jsonCodec) ParseRequestArguments(argTypes []reflect.Type, params interface{}) ([]reflect.Value, Error) {
if args, ok := params.(json.RawMessage); !ok {
return nil, &invalidParamsError{"Invalid params supplied"}
} else {
return parsePositionalArguments(args, argTypes)
}
}
// parsePositionalArguments tries to parse the given args to an array of values with the
// given types. It returns the parsed values or an error when the args could not be
// parsed. Missing optional arguments are returned as reflect.Zero values.
func parsePositionalArguments(rawArgs json.RawMessage, types []reflect.Type) ([]reflect.Value, Error) {
// Read beginning of the args array.
func parsePositionalArguments(rawArgs json.RawMessage, types []reflect.Type) ([]reflect.Value, error) {
dec := json.NewDecoder(bytes.NewReader(rawArgs))
if tok, _ := dec.Token(); tok != json.Delim('[') {
return nil, &invalidParamsError{"non-array args"}
}
// Read args.
args := make([]reflect.Value, 0, len(types))
for i := 0; dec.More(); i++ {
if i >= len(types) {
return nil, &invalidParamsError{fmt.Sprintf("too many arguments, want at most %d", len(types))}
var args []reflect.Value
tok, err := dec.Token()
switch {
case err == io.EOF || tok == nil && err == nil:
// "params" is optional and may be empty. Also allow "params":null even though it's
// not in the spec because our own client used to send it.
case err != nil:
return nil, err
case tok == json.Delim('['):
// Read argument array.
if args, err = parseArgumentArray(dec, types); err != nil {
return nil, err
}
argval := reflect.New(types[i])
if err := dec.Decode(argval.Interface()); err != nil {
return nil, &invalidParamsError{fmt.Sprintf("invalid argument %d: %v", i, err)}
}
if argval.IsNil() && types[i].Kind() != reflect.Ptr {
return nil, &invalidParamsError{fmt.Sprintf("missing value for required argument %d", i)}
}
args = append(args, argval.Elem())
}
// Read end of args array.
if _, err := dec.Token(); err != nil {
return nil, &invalidParamsError{err.Error()}
default:
return nil, errors.New("non-array args")
}
// Set any missing args to nil.
for i := len(args); i < len(types); i++ {
if types[i].Kind() != reflect.Ptr {
return nil, &invalidParamsError{fmt.Sprintf("missing value for required argument %d", i)}
return nil, fmt.Errorf("missing value for required argument %d", i)
}
args = append(args, reflect.Zero(types[i]))
}
return args, nil
}
// CreateResponse will create a JSON-RPC success response with the given id and reply as result.
func (c *jsonCodec) CreateResponse(id interface{}, reply interface{}) interface{} {
return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: reply}
func parseArgumentArray(dec *json.Decoder, types []reflect.Type) ([]reflect.Value, error) {
args := make([]reflect.Value, 0, len(types))
for i := 0; dec.More(); i++ {
if i >= len(types) {
return args, fmt.Errorf("too many arguments, want at most %d", len(types))
}
argval := reflect.New(types[i])
if err := dec.Decode(argval.Interface()); err != nil {
return args, fmt.Errorf("invalid argument %d: %v", i, err)
}
if argval.IsNil() && types[i].Kind() != reflect.Ptr {
return args, fmt.Errorf("missing value for required argument %d", i)
}
args = append(args, argval.Elem())
}
// Read end of args array.
_, err := dec.Token()
return args, err
}
// CreateErrorResponse will create a JSON-RPC error response with the given id and error.
func (c *jsonCodec) CreateErrorResponse(id interface{}, err Error) interface{} {
return &jsonErrResponse{Version: jsonrpcVersion, Id: id, Error: jsonError{Code: err.ErrorCode(), Message: err.Error()}}
}
// CreateErrorResponseWithInfo will create a JSON-RPC error response with the given id and error.
// info is optional and contains additional information about the error. When an empty string is passed it is ignored.
func (c *jsonCodec) CreateErrorResponseWithInfo(id interface{}, err Error, info interface{}) interface{} {
return &jsonErrResponse{Version: jsonrpcVersion, Id: id,
Error: jsonError{Code: err.ErrorCode(), Message: err.Error(), Data: info}}
}
// CreateNotification will create a JSON-RPC notification with the given subscription id and event as params.
func (c *jsonCodec) CreateNotification(subid, namespace string, event interface{}) interface{} {
return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix,
Params: jsonSubscription{Subscription: subid, Result: event}}
}
// Write message to client
func (c *jsonCodec) Write(res interface{}) error {
c.encMu.Lock()
defer c.encMu.Unlock()
return c.encode(res)
}
// Close the underlying connection
func (c *jsonCodec) Close() {
c.closer.Do(func() {
close(c.closed)
c.rw.Close()
})
}
// Closed returns a channel which will be closed when Close is called
func (c *jsonCodec) Closed() <-chan interface{} {
return c.closed
// parseSubscriptionName extracts the subscription name from an encoded argument array.
func parseSubscriptionName(rawArgs json.RawMessage) (string, error) {
dec := json.NewDecoder(bytes.NewReader(rawArgs))
if tok, _ := dec.Token(); tok != json.Delim('[') {
return "", errors.New("non-array args")
}
v, _ := dec.Token()
method, ok := v.(string)
if !ok {
return "", errors.New("expected subscription name as first argument")
}
return method, nil
}