gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Optimize RPC pubsub for multiple clients with the same subscription (#18943)

Browse Source 
* reimplement rpc pubsub with a broadcast queue

* update tests for new pubsub implementation

* fix: fix review suggestions

* chore(rpc): add additional pubsub metrics

* integrate max subscriptions check into SubscriptionTracker to reduce locking

* separate subscription control from tracker

* limit memory usage of items in pubsub broadcast queue, improve error handling

* add more pubsub metrics

* add final count metrics to pubsub

* add metric for total number of subscriptions

* fix small review suggestions

* remove by_params from SubscriptionTracker and add node_progress_watchers map instead

* add subscription tracker tests

* add metrics for number of pubsub notifications as a counter

* ignore clippy lint in TokenCounter

* fix underflow in token counter

* reduce queue capacity in pubsub tests

* fix(rpc): fix test timeouts

* fix race in account subscription test

* Add RpcSubscriptions::new_for_tests

Co-authored-by: Pavel Strakhov <p.strakhov@iconic.vc>
Co-authored-by: Nikita Podoliako <n.podoliako@zubr.io>
Co-authored-by: Tyera Eulberg <tyera@solana.com>
This commit is contained in:
Pavel Strakhov
2021-09-17 22:40:14 +03:00
committed by GitHub
parent fc2bf2d3b6
commit 65227f44dc
22 changed files with 2555 additions and 2073 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/Cargo.toml
View File

@ -64,6 +64,8 @@ trees = "0.4.2"
[dev-dependencies]
jsonrpc-core = "18.0.0"
jsonrpc-core-client = { version = "18.0.0", features = ["ipc", "ws"] }
jsonrpc-derive = "18.0.0"
jsonrpc-pubsub = "18.0.0"
matches = "0.1.9"
reqwest = { version = "0.11.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
serde_json = "1.0.68"
@ -74,6 +76,7 @@ solana-version = { path = "../version", version = "=1.8.0" }
static_assertions = "1.1.0"
systemstat = "0.1.8"
[build-dependencies]
rustc_version = "0.4"
@ -96,4 +99,4 @@ name = "sigverify_stage"
name = "retransmit_stage"
[package.metadata.docs.rs]
targets = ["x86_64-unknown-linux-gnu"]
targets = ["x86_64-unknown-linux-gnu"]

4
core/src/cluster_info_vote_listener.rs
View File

@ -1530,7 +1530,7 @@ mod tests {
let vote_tracker = VoteTracker::new(&bank);
let optimistically_confirmed_bank =
OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
let subscriptions = Arc::new(RpcSubscriptions::new(
let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks,
Arc::new(RwLock::new(BlockCommitmentCache::default())),
@ -1649,7 +1649,7 @@ mod tests {
let bank = bank_forks.read().unwrap().get(0).unwrap().clone();
let optimistically_confirmed_bank =
OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
let subscriptions = Arc::new(RpcSubscriptions::new(
let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks,
Arc::new(RwLock::new(BlockCommitmentCache::default())),

6
core/src/replay_stage.rs
View File

@ -3014,7 +3014,7 @@ pub mod tests {
let optimistically_confirmed_bank =
OptimisticallyConfirmedBank::locked_from_bank_forks_root(bank_forks);
let exit = Arc::new(AtomicBool::new(false));
let rpc_subscriptions = Arc::new(RpcSubscriptions::new(
let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks.clone(),
Arc::new(RwLock::new(BlockCommitmentCache::default())),
@ -3545,7 +3545,7 @@ pub mod tests {
&replay_vote_sender,
&VerifyRecyclers::default(),
);
let rpc_subscriptions = Arc::new(RpcSubscriptions::new(
let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks.clone(),
block_commitment_cache,
@ -3613,7 +3613,7 @@ pub mod tests {
let exit = Arc::new(AtomicBool::new(false));
let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
let rpc_subscriptions = Arc::new(RpcSubscriptions::new(
let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks.clone(),
block_commitment_cache.clone(),

2
core/src/tvu.rs
View File

@ -452,7 +452,7 @@ pub mod tests {
},
blockstore,
ledger_signal_receiver,
&Arc::new(RpcSubscriptions::new(
&Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks.clone(),
block_commitment_cache.clone(),

16
core/src/validator.rs
View File

@ -471,12 +471,12 @@ impl Validator {
let optimistically_confirmed_bank =
OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
let rpc_subscriptions = Arc::new(RpcSubscriptions::new_with_vote_subscription(
let rpc_subscriptions = Arc::new(RpcSubscriptions::new_with_config(
&exit,
bank_forks.clone(),
block_commitment_cache.clone(),
optimistically_confirmed_bank.clone(),
config.pubsub_config.enable_vote_subscription,
&config.pubsub_config,
));
let max_slots = Arc::new(MaxSlots::default());
@ -577,12 +577,18 @@ impl Validator {
if config.rpc_config.minimal_api {
None
} else {
Some(PubSubService::new(
let (trigger, pubsub_service) = PubSubService::new(
config.pubsub_config.clone(),
&rpc_subscriptions,
rpc_pubsub_addr,
&exit,
))
);
config
.validator_exit
.write()
.unwrap()
.register_exit(Box::new(move || trigger.cancel()));
Some(pubsub_service)
},
Some(OptimisticallyConfirmedBankTracker::new(
bank_notification_receiver,

7
core/tests/client.rs
View File

@ -98,14 +98,14 @@ fn test_slot_subscription() {
let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
let optimistically_confirmed_bank =
OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
let subscriptions = Arc::new(RpcSubscriptions::new(
let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
&exit,
bank_forks,
Arc::new(RwLock::new(BlockCommitmentCache::default())),
optimistically_confirmed_bank,
));
let pubsub_service =
PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr, &exit);
let (trigger, pubsub_service) =
PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr);
std::thread::sleep(Duration::from_millis(400));
let (mut client, receiver) =
@ -138,6 +138,7 @@ fn test_slot_subscription() {
}
exit.store(true, Ordering::Relaxed);
trigger.cancel();
client.shutdown().unwrap();
pubsub_service.close().unwrap();

8
core/tests/rpc.rs
View File

@ -1,6 +1,7 @@
use bincode::serialize;
use jsonrpc_core::futures::StreamExt;
use jsonrpc_core_client::transports::ws;
use log::*;
use reqwest::{self, header::CONTENT_TYPE};
use serde_json::{json, Value};
@ -10,11 +11,12 @@ use solana_client::{
rpc_client::RpcClient,
rpc_config::{RpcAccountInfoConfig, RpcSignatureSubscribeConfig},
rpc_request::RpcError,
rpc_response::{Response, RpcSignatureResult, SlotUpdate},
rpc_response::{Response as RpcResponse, RpcSignatureResult, SlotUpdate},
tpu_client::{TpuClient, TpuClientConfig},
};
use solana_core::test_validator::TestValidator;
use solana_rpc::rpc_pubsub::gen_client::Client as PubsubClient;
use solana_sdk::{
commitment_config::CommitmentConfig,
hash::Hash,
@ -256,9 +258,9 @@ fn test_rpc_subscriptions() {
// Track when subscriptions are ready
let (ready_sender, ready_receiver) = channel::<()>();
// Track account notifications are received
let (account_sender, account_receiver) = channel::<Response<UiAccount>>();
let (account_sender, account_receiver) = channel::<RpcResponse<UiAccount>>();
// Track when status notifications are received
let (status_sender, status_receiver) = channel::<(String, Response<RpcSignatureResult>)>();
let (status_sender, status_receiver) = channel::<(String, RpcResponse<RpcSignatureResult>)>();
// Create the pub sub runtime
let rt = Runtime::new().unwrap();