rpc: metrics for JSON-RPC method calls (#20847)

This adds a couple of metrics for tracking the timing
and frequency of method calls:

- rpc/requests gauge counts all requests
- rpc/success gauge counts requests which return err == nil
- rpc/failure gauge counts requests which return err != nil
- rpc/duration/all timer tracks timing of all requests
- rpc/duration/<method>/<success/failure> tracks per-method timing

rpc/handler.go

@@ -327,8 +327,22 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage
 	if err != nil {
 		return msg.errorResponse(&invalidParamsError{err.Error()})
 	}
+	start := time.Now()
+	answer := h.runMethod(cp.ctx, msg, callb, args)
 
-	return h.runMethod(cp.ctx, msg, callb, args)
+	// Collect the statistics for RPC calls if metrics is enabled.
+	// We only care about pure rpc call. Filter out subscription.
+	if callb != h.unsubscribeCb {
+		rpcRequestGauge.Inc(1)
+		if answer.Error != nil {
+			failedReqeustGauge.Inc(1)
+		} else {
+			successfulRequestGauge.Inc(1)
+		}
+		rpcServingTimer.UpdateSince(start)
+		newRPCServingTimer(msg.Method, answer.Error == nil).UpdateSince(start)
+	}
+	return answer
 }
 
 // handleSubscribe processes *_subscribe method calls.