Fix the occasional stuck RPC request (bp #14628) (#14638)

* WIP fix the occasional stuck RPC request

(cherry picked from commit 5cf9094bb9)

* Clean up and add comment

(cherry picked from commit 8d4ab1bab1)

Co-authored-by: Ryo Onodera <ryoqun@gmail.com>

core/src/rpc_service.rs

@@ -358,6 +358,20 @@ impl JsonRpcService {
 
         let ledger_path = ledger_path.to_path_buf();
 
+        // sadly, some parts of our current rpc implemention block the jsonrpc's
+        // _socket-listening_ event loop for too long, due to (blocking) long IO or intesive CPU,
+        // causing no further processing of incoming requests and ultimatily innocent clients timing-out.
+        // So create a (shared) multi-threaded event_loop for jsonrpc and set its .threads() to 1,
+        // so that we avoid the single-threaded event loops from being created automatically by
+        // jsonrpc for threads when .threads(N > 1) is given.
+        let event_loop = {
+            tokio_01::runtime::Builder::new()
+                .core_threads(rpc_threads)
+                .name_prefix("sol-rpc-el")
+                .build()
+                .unwrap()
+        };
+
         let (close_handle_sender, close_handle_receiver) = channel();
         let thread_hdl = Builder::new()
             .name("solana-jsonrpc".to_string())
@@ -376,7 +390,8 @@ impl JsonRpcService {
                     io,
                     move |_req: &hyper::Request<hyper::Body>| request_processor.clone(),
                 )
-                .threads(rpc_threads)
+                .event_loop_executor(event_loop.executor())
+                .threads(1)
                 .cors(DomainsValidation::AllowOnly(vec![
                     AccessControlAllowOrigin::Any,
                 ]))