Optimize RPC pubsub for multiple clients with the same subscription (#18943)

* reimplement rpc pubsub with a broadcast queue

* update tests for new pubsub implementation

* fix: fix review suggestions

* chore(rpc): add additional pubsub metrics

* integrate max subscriptions check into SubscriptionTracker to reduce locking

* separate subscription control from tracker

* limit memory usage of items in pubsub broadcast queue, improve error handling

* add more pubsub metrics

* add final count metrics to pubsub

* add metric for total number of subscriptions

* fix small review suggestions

* remove by_params from SubscriptionTracker and add node_progress_watchers map instead

* add subscription tracker tests

* add metrics for number of pubsub notifications as a counter

* ignore clippy lint in TokenCounter

* fix underflow in token counter

* reduce queue capacity in pubsub tests

* fix(rpc): fix test timeouts

* fix race in account subscription test

* Add RpcSubscriptions::new_for_tests

Co-authored-by: Pavel Strakhov <p.strakhov@iconic.vc>
Co-authored-by: Nikita Podoliako <n.podoliako@zubr.io>
Co-authored-by: Tyera Eulberg <tyera@solana.com>

replica-node/src/replica_node.rs

@@ -208,6 +208,17 @@ fn start_client_rpc_services(
         ));
     }
 
+    let (trigger, pubsub_service) = PubSubService::new(
+        replica_config.pubsub_config.clone(),
+        &subscriptions,
+        replica_config.rpc_pubsub_addr,
+    );
+    replica_config
+        .replica_exit
+        .write()
+        .unwrap()
+        .register_exit(Box::new(move || trigger.cancel()));
+
     let (_bank_notification_sender, bank_notification_receiver) = unbounded();
     (
         Some(JsonRpcService::new(
@@ -231,18 +242,13 @@ fn start_client_rpc_services(
             leader_schedule_cache.clone(),
             max_complete_transaction_status_slot,
         )),
-        Some(PubSubService::new(
-            replica_config.pubsub_config.clone(),
-            &subscriptions,
-            replica_config.rpc_pubsub_addr,
-            &exit,
-        )),
+        Some(pubsub_service),
         Some(OptimisticallyConfirmedBankTracker::new(
             bank_notification_receiver,
             &exit,
             bank_forks.clone(),
             optimistically_confirmed_bank.clone(),
-            subscriptions.clone(),
+            subscriptions,
             None,
         )),
     )