From b7fc704092c6b931632b9263cd203f2fc94fa24d Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 20 Sep 2021 07:32:34 +0000 Subject: [PATCH] Optimize RPC pubsub for multiple clients with the same subscription (backport #18943) (#19986) * Optimize RPC pubsub for multiple clients with the same subscription (#18943) * reimplement rpc pubsub with a broadcast queue * update tests for new pubsub implementation * fix: fix review suggestions * chore(rpc): add additional pubsub metrics * integrate max subscriptions check into SubscriptionTracker to reduce locking * separate subscription control from tracker * limit memory usage of items in pubsub broadcast queue, improve error handling * add more pubsub metrics * add final count metrics to pubsub * add metric for total number of subscriptions * fix small review suggestions * remove by_params from SubscriptionTracker and add node_progress_watchers map instead * add subscription tracker tests * add metrics for number of pubsub notifications as a counter * ignore clippy lint in TokenCounter * fix underflow in token counter * reduce queue capacity in pubsub tests * fix(rpc): fix test timeouts * fix race in account subscription test * Add RpcSubscriptions::new_for_tests Co-authored-by: Pavel Strakhov Co-authored-by: Nikita Podoliako Co-authored-by: Tyera Eulberg (cherry picked from commit 65227f44dc7dd7ce8b41262de5962409a1eb9042) * Fix conflicts (and standardize naming to make future subscription backports easier) Co-authored-by: Pavel Strakhov Co-authored-by: Tyera Eulberg --- Cargo.lock | 60 +- account-decoder/src/lib.rs | 4 +- client/src/rpc_filter.rs | 8 +- core/Cargo.toml | 7 +- core/src/cluster_info_vote_listener.rs | 4 +- core/src/lib.rs | 1 + .../optimistically_confirmed_bank_tracker.rs | 2 +- core/src/replay_stage.rs | 54 +- core/src/rpc.rs | 2 +- core/src/rpc_pubsub.rs | 949 +++---- core/src/rpc_pubsub_service.rs | 366 ++- core/src/rpc_subscription_tracker.rs | 731 +++++ core/src/rpc_subscriptions.rs | 2501 +++++++---------- core/src/tvu.rs | 8 +- core/src/validator.rs | 26 +- core/tests/client.rs | 7 +- core/tests/rpc.rs | 6 +- metrics/src/lib.rs | 58 + replica-node/src/replica_node.rs | 339 +++ sdk/src/commitment_config.rs | 2 +- validator/src/main.rs | 72 +- 21 files changed, 2993 insertions(+), 2214 deletions(-) create mode 100644 core/src/rpc_subscription_tracker.rs create mode 100644 replica-node/src/replica_node.rs diff --git a/Cargo.lock b/Cargo.lock index bf152dc7a3..177c72d039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,6 +763,15 @@ dependencies = [ "winapi 0.2.8", ] +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -2187,9 +2196,9 @@ checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" [[package]] name = "libc" -version = "0.2.98" +version = "0.2.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790" +checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103" [[package]] name = "libloading" @@ -2734,7 +2743,7 @@ dependencies = [ "mio 0.6.22", "mio-extras", "rand 0.7.3", - "sha-1", + "sha-1 0.8.2", "slab", "url 2.2.0", ] @@ -3840,6 +3849,19 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "sha-1" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + [[package]] name = "sha1" version = "0.6.0" @@ -3980,6 +4002,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "soketto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74e48087dbeed4833785c2f3352b59140095dc192dce966a3bfc155020a439f" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "futures 0.3.8", + "httparse", + "log 0.4.11", + "rand 0.8.3", + "sha-1 0.9.8", +] + [[package]] name = "solana-account-decoder" version = "1.6.27" @@ -4395,6 +4432,7 @@ dependencies = [ "chrono", "core_affinity", "crossbeam-channel 0.4.4", + "dashmap", "ed25519-dalek", "flate2", "fs_extra", @@ -4428,6 +4466,7 @@ dependencies = [ "serde_derive", "serde_json", "serial_test", + "soketto", "solana-account-decoder", "solana-banks-server", "solana-clap-utils", @@ -4454,6 +4493,7 @@ dependencies = [ "solana-version", "solana-vote-program", "spl-token", + "stream-cancel", "symlink", "systemstat", "tempfile", @@ -5748,6 +5788,17 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +[[package]] +name = "stream-cancel" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0a9eb2715209fb8cc0d942fcdff45674bfc9f0090a0d897e85a22955ad159b" +dependencies = [ + "futures-core", + "pin-project 1.0.1", + "tokio", +] + [[package]] name = "strsim" version = "0.8.0" @@ -6244,6 +6295,7 @@ checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" dependencies = [ "bytes 1.0.1", "futures-core", + "futures-io", "futures-sink", "log 0.4.11", "pin-project-lite", @@ -6420,7 +6472,7 @@ dependencies = [ "log 0.4.11", "native-tls", "rand 0.7.3", - "sha-1", + "sha-1 0.8.2", "url 2.2.0", "utf-8", ] diff --git a/account-decoder/src/lib.rs b/account-decoder/src/lib.rs index 7f1e7c40c7..a590955979 100644 --- a/account-decoder/src/lib.rs +++ b/account-decoder/src/lib.rs @@ -48,7 +48,7 @@ pub enum UiAccountData { Binary(String, UiAccountEncoding), } -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] pub enum UiAccountEncoding { Binary, // Legacy. Retained for RPC backwards compatibility @@ -166,7 +166,7 @@ impl Default for UiFeeCalculator { } } -#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UiDataSliceConfig { pub offset: usize, diff --git a/client/src/rpc_filter.rs b/client/src/rpc_filter.rs index 500f74f9c0..eeb54e5dcf 100644 --- a/client/src/rpc_filter.rs +++ b/client/src/rpc_filter.rs @@ -1,6 +1,6 @@ use thiserror::Error; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum RpcFilterType { DataSize(u64), @@ -40,19 +40,19 @@ pub enum RpcFilterError { Base58DataTooLarge, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum MemcmpEncoding { Binary, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase", untagged)] pub enum MemcmpEncodedBytes { Binary(String), } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct Memcmp { /// Data offset to begin match pub offset: usize, diff --git a/core/Cargo.toml b/core/Cargo.toml index 9adb6e2d00..1b3e87b011 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -24,6 +24,7 @@ byteorder = "1.3.4" chrono = { version = "0.4.11", features = ["serde"] } core_affinity = "0.5.10" crossbeam-channel = "0.4" +dashmap = "4.0.2" ed25519-dalek = "=1.0.1" fs_extra = "1.2.0" flate2 = "1.0" @@ -52,6 +53,7 @@ serde = "1.0.122" serde_bytes = "0.11" serde_derive = "1.0.103" serde_json = "1.0.56" +soketto = "0.6" solana-account-decoder = { path = "../account-decoder", version = "=1.6.27" } solana-banks-server = { path = "../banks-server", version = "=1.6.27" } solana-clap-utils = { path = "../clap-utils", version = "=1.6.27" } @@ -78,13 +80,16 @@ solana-version = { path = "../version", version = "=1.6.27" } solana-vote-program = { path = "../programs/vote", version = "=1.6.27" } spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] } tempfile = "3.1.0" +stream-cancel = "0.8.1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.6", features = ["codec"] } +tokio-util = { version = "0.6", features = ["codec", "compat"] } solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.6.27" } trees = "0.2.1" [dev-dependencies] +jsonrpc-derive = "18.0.0" +jsonrpc-pubsub = "18.0.0" matches = "0.1.6" num_cpus = "1.13.0" reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] } diff --git a/core/src/cluster_info_vote_listener.rs b/core/src/cluster_info_vote_listener.rs index 20a4b15aed..5d2262ba90 100644 --- a/core/src/cluster_info_vote_listener.rs +++ b/core/src/cluster_info_vote_listener.rs @@ -1522,7 +1522,7 @@ mod tests { let vote_tracker = VoteTracker::new(&bank); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), @@ -1641,7 +1641,7 @@ mod tests { let bank = bank_forks.read().unwrap().get(0).unwrap().clone(); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), diff --git a/core/src/lib.rs b/core/src/lib.rs index 3be06e630b..37ae6b040a 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -66,6 +66,7 @@ pub mod rpc_health; pub mod rpc_pubsub; pub mod rpc_pubsub_service; pub mod rpc_service; +pub mod rpc_subscription_tracker; pub mod rpc_subscriptions; pub mod send_transaction_service; pub mod serve_repair; diff --git a/core/src/optimistically_confirmed_bank_tracker.rs b/core/src/optimistically_confirmed_bank_tracker.rs index 6303e843b0..b37c859c2a 100644 --- a/core/src/optimistically_confirmed_bank_tracker.rs +++ b/core/src/optimistically_confirmed_bank_tracker.rs @@ -296,7 +296,7 @@ mod tests { OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache, diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 057733ac47..5ab1cd3ed2 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -115,7 +115,7 @@ pub struct ReplayStageConfig { pub vote_account: Pubkey, pub authorized_voter_keypairs: Arc>>>, pub exit: Arc, - pub subscriptions: Arc, + pub rpc_subscriptions: Arc, pub leader_schedule_cache: Arc, pub latest_root_senders: Vec>, pub accounts_background_request_sender: AbsRequestSender, @@ -313,7 +313,7 @@ impl ReplayStage { vote_account, authorized_voter_keypairs, exit, - subscriptions, + rpc_subscriptions, leader_schedule_cache, latest_root_senders, accounts_background_request_sender, @@ -330,7 +330,7 @@ impl ReplayStage { let (lockouts_sender, commitment_service) = AggregateCommitmentService::new( &exit, block_commitment_cache.clone(), - subscriptions.clone(), + rpc_subscriptions.clone(), ); #[allow(clippy::cognitive_complexity)] @@ -378,7 +378,7 @@ impl ReplayStage { &blockstore, &bank_forks, &leader_schedule_cache, - &subscriptions, + &rpc_subscriptions, &mut progress, ); generate_new_bank_forks_time.stop(); @@ -403,7 +403,7 @@ impl ReplayStage { &replay_vote_sender, &bank_notification_sender, &rewards_recorder_sender, - &subscriptions, + &rpc_subscriptions, &mut duplicate_slots_tracker, &gossip_duplicate_confirmed_slots, &ancestors, @@ -597,7 +597,7 @@ impl ReplayStage { &lockouts_sender, &accounts_background_request_sender, &latest_root_senders, - &subscriptions, + &rpc_subscriptions, &block_commitment_cache, &mut heaviest_subtree_fork_choice, &bank_notification_sender, @@ -695,7 +695,7 @@ impl ReplayStage { &bank_forks, &poh_recorder, &leader_schedule_cache, - &subscriptions, + &rpc_subscriptions, &progress, &retransmit_slots_sender, &mut skipped_slots_info, @@ -1127,7 +1127,7 @@ impl ReplayStage { bank_forks: &Arc>, poh_recorder: &Arc>, leader_schedule_cache: &Arc, - subscriptions: &Arc, + rpc_subscriptions: &Arc, progress_map: &ProgressMap, retransmit_slots_sender: &RetransmitSlotsSender, skipped_slots_info: &mut SkippedSlotsInfo, @@ -1234,7 +1234,7 @@ impl ReplayStage { poh_slot, root_slot, my_pubkey, - subscriptions, + rpc_subscriptions, root_distance > 500, ); @@ -1284,7 +1284,7 @@ impl ReplayStage { bank: &Bank, root: Slot, err: &BlockstoreProcessorError, - subscriptions: &Arc, + rpc_subscriptions: &Arc, duplicate_slots_tracker: &mut DuplicateSlotsTracker, gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, ancestors: &HashMap>, @@ -1320,7 +1320,7 @@ impl ReplayStage { blockstore .set_dead_slot(slot) .expect("Failed to mark slot as dead in blockstore"); - subscriptions.notify_slot_update(SlotUpdate::Dead { + rpc_subscriptions.notify_slot_update(SlotUpdate::Dead { slot, err: format!("error: {:?}", err), timestamp: timestamp(), @@ -1354,7 +1354,7 @@ impl ReplayStage { lockouts_sender: &Sender, accounts_background_request_sender: &AbsRequestSender, latest_root_senders: &[Sender], - subscriptions: &Arc, + rpc_subscriptions: &Arc, block_commitment_cache: &Arc>, heaviest_subtree_fork_choice: &mut HeaviestSubtreeForkChoice, bank_notification_sender: &Option, @@ -1415,7 +1415,7 @@ impl ReplayStage { has_new_vote_been_rooted, vote_signatures, ); - subscriptions.notify_roots(rooted_slots); + rpc_subscriptions.notify_roots(rooted_slots); if let Some(sender) = bank_notification_sender { sender .send(BankNotification::Root(root_bank)) @@ -1711,7 +1711,7 @@ impl ReplayStage { replay_vote_sender: &ReplayVoteSender, bank_notification_sender: &Option, rewards_recorder_sender: &Option, - subscriptions: &Arc, + rpc_subscriptions: &Arc, duplicate_slots_tracker: &mut DuplicateSlotsTracker, gossip_duplicate_confirmed_slots: &GossipDuplicateConfirmedSlots, ancestors: &HashMap>, @@ -1783,7 +1783,7 @@ impl ReplayStage { &bank, root_slot, &err, - subscriptions, + rpc_subscriptions, duplicate_slots_tracker, gossip_duplicate_confirmed_slots, ancestors, @@ -2446,7 +2446,7 @@ impl ReplayStage { blockstore: &Blockstore, bank_forks: &RwLock, leader_schedule_cache: &Arc, - subscriptions: &Arc, + rpc_subscriptions: &Arc, progress: &mut ProgressMap, ) { // Find the next slot that chains to the old slot @@ -2491,7 +2491,7 @@ impl ReplayStage { child_slot, forks.root(), &leader, - subscriptions, + rpc_subscriptions, false, ); let empty: Vec = vec![]; @@ -2518,10 +2518,10 @@ impl ReplayStage { slot: u64, root_slot: u64, leader: &Pubkey, - subscriptions: &Arc, + rpc_subscriptions: &Arc, vote_only_bank: bool, ) -> Bank { - subscriptions.notify_slot(slot, parent.slot(), root_slot); + rpc_subscriptions.notify_slot(slot, parent.slot(), root_slot); Bank::new_from_parent_with_vote_only(parent, leader, slot, vote_only_bank) } @@ -2721,7 +2721,7 @@ pub(crate) mod tests { let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let exit = Arc::new(AtomicBool::new(false)); - let rpc_subscriptions = Arc::new(RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::default())), @@ -3239,8 +3239,7 @@ pub(crate) mod tests { &replay_vote_sender, &&VerifyRecyclers::default(), ); - - let subscriptions = Arc::new(RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache, @@ -3252,7 +3251,7 @@ pub(crate) mod tests { &bank0, 0, err, - &subscriptions, + &rpc_subscriptions, &mut DuplicateSlotsTracker::default(), &GossipDuplicateConfirmedSlots::default(), &HashMap::new(), @@ -3305,14 +3304,17 @@ pub(crate) mod tests { let exit = Arc::new(AtomicBool::new(false)); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default())); - let subscriptions = Arc::new(RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache.clone(), OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), )); - let (lockouts_sender, _) = - AggregateCommitmentService::new(&exit, block_commitment_cache.clone(), subscriptions); + let (lockouts_sender, _) = AggregateCommitmentService::new( + &exit, + block_commitment_cache.clone(), + rpc_subscriptions, + ); assert!(block_commitment_cache .read() diff --git a/core/src/rpc.rs b/core/src/rpc.rs index b653206435..d508394af7 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -7338,7 +7338,7 @@ pub mod tests { OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache.clone(), diff --git a/core/src/rpc_pubsub.rs b/core/src/rpc_pubsub.rs index b77b68de88..ee099f3942 100644 --- a/core/src/rpc_pubsub.rs +++ b/core/src/rpc_pubsub.rs @@ -1,10 +1,19 @@ //! The `pubsub` module implements a threaded subscription service on client RPC request -use crate::rpc_subscriptions::{RpcSubscriptions, RpcVote}; +use crate::{ + rpc_pubsub_service::PubSubConfig, + rpc_subscription_tracker::{ + AccountSubscriptionParams, LogsSubscriptionKind, LogsSubscriptionParams, + ProgramSubscriptionParams, SignatureSubscriptionParams, SubscriptionControl, + SubscriptionId, SubscriptionParams, SubscriptionToken, + }, + rpc_subscriptions::RpcVote, +}; +use dashmap::DashMap; use jsonrpc_core::{Error, ErrorCode, Result}; use jsonrpc_derive::rpc; -use jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId}; -use solana_account_decoder::UiAccount; +use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId as PubSubSubscriptionId}; +use solana_account_decoder::{UiAccount, UiAccountEncoding}; use solana_client::{ rpc_config::{ RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig, @@ -15,17 +24,16 @@ use solana_client::{ SlotUpdate, }, }; -#[cfg(test)] -use solana_runtime::bank_forks::BankForks; use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}; -#[cfg(test)] -use std::sync::RwLock; -use std::{ - str::FromStr, - sync::{atomic, Arc}, -}; +use std::{str::FromStr, sync::Arc}; -pub const MAX_ACTIVE_SUBSCRIPTIONS: usize = 100_000; +// We have to keep both of the following traits to not break backwards compatibility. +// `RpcSolPubSubInternal` is actually used by the current PubSub API implementation. +// `RpcSolPubSub` and the corresponding `gen_client` module are preserved +// so the clients reliant on `gen_client::Client` do not break after this implementation is released. +// +// There are no compile-time checks that ensure coherence between traits +// so extra attention is required when adding a new method to the API. // Suppress needless_return due to // https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204 @@ -56,8 +64,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "accountUnsubscribe" )] - fn account_unsubscribe(&self, meta: Option, id: SubscriptionId) - -> Result; + fn account_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get notification every time account data owned by a particular program is changed // Accepts pubkey parameter as base-58 encoded string @@ -80,8 +91,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "programUnsubscribe" )] - fn program_unsubscribe(&self, meta: Option, id: SubscriptionId) - -> Result; + fn program_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get logs for all transactions that reference the specified address #[pubsub(subscription = "logsNotification", subscribe, name = "logsSubscribe")] @@ -99,7 +113,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "logsUnsubscribe" )] - fn logs_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn logs_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get notification when signature is verified // Accepts signature parameter as base-58 encoded string @@ -125,7 +143,7 @@ pub trait RpcSolPubSub { fn signature_unsubscribe( &self, meta: Option, - id: SubscriptionId, + id: PubSubSubscriptionId, ) -> Result; // Get notification when slot is encountered @@ -138,7 +156,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "slotUnsubscribe" )] - fn slot_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn slot_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get series of updates for all slots #[pubsub( @@ -161,7 +183,7 @@ pub trait RpcSolPubSub { fn slots_updates_unsubscribe( &self, meta: Option, - id: SubscriptionId, + id: PubSubSubscriptionId, ) -> Result; // Get notification when vote is encountered @@ -174,7 +196,11 @@ pub trait RpcSolPubSub { unsubscribe, name = "voteUnsubscribe" )] - fn vote_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn vote_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; // Get notification when a new root is set #[pubsub(subscription = "rootNotification", subscribe, name = "rootSubscribe")] @@ -186,53 +212,149 @@ pub trait RpcSolPubSub { unsubscribe, name = "rootUnsubscribe" )] - fn root_unsubscribe(&self, meta: Option, id: SubscriptionId) -> Result; + fn root_unsubscribe( + &self, + meta: Option, + id: PubSubSubscriptionId, + ) -> Result; +} + +pub use internal::RpcSolPubSubInternal; + +// We have to use a separate module so the code generated by different `rpc` macro invocations do not interfere with each other. +mod internal { + use super::*; + + #[rpc] + pub trait RpcSolPubSubInternal { + // Get notification every time account data is changed + // Accepts pubkey parameter as base-58 encoded string + #[rpc(name = "accountSubscribe")] + fn account_subscribe( + &self, + pubkey_str: String, + config: Option, + ) -> Result; + + // Unsubscribe from account notification subscription. + #[rpc(name = "accountUnsubscribe")] + fn account_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification every time account data owned by a particular program is changed + // Accepts pubkey parameter as base-58 encoded string + #[rpc(name = "programSubscribe")] + fn program_subscribe( + &self, + pubkey_str: String, + config: Option, + ) -> Result; + + // Unsubscribe from account notification subscription. + #[rpc(name = "programUnsubscribe")] + fn program_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get logs for all transactions that reference the specified address + #[rpc(name = "logsSubscribe")] + fn logs_subscribe( + &self, + filter: RpcTransactionLogsFilter, + config: Option, + ) -> Result; + + // Unsubscribe from logs notification subscription. + #[rpc(name = "logsUnsubscribe")] + fn logs_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when signature is verified + // Accepts signature parameter as base-58 encoded string + #[rpc(name = "signatureSubscribe")] + fn signature_subscribe( + &self, + signature_str: String, + config: Option, + ) -> Result; + + // Unsubscribe from signature notification subscription. + #[rpc(name = "signatureUnsubscribe")] + fn signature_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when slot is encountered + #[rpc(name = "slotSubscribe")] + fn slot_subscribe(&self) -> Result; + + // Unsubscribe from slot notification subscription. + #[rpc(name = "slotUnsubscribe")] + fn slot_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get series of updates for all slots + #[rpc(name = "slotsUpdatesSubscribe")] + fn slots_updates_subscribe(&self) -> Result; + + // Unsubscribe from slots updates notification subscription. + #[rpc(name = "slotsUpdatesUnsubscribe")] + fn slots_updates_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when vote is encountered + #[rpc(name = "voteSubscribe")] + fn vote_subscribe(&self) -> Result; + + // Unsubscribe from vote notification subscription. + #[rpc(name = "voteUnsubscribe")] + fn vote_unsubscribe(&self, id: SubscriptionId) -> Result; + + // Get notification when a new root is set + #[rpc(name = "rootSubscribe")] + fn root_subscribe(&self) -> Result; + + // Unsubscribe from slot notification subscription. + #[rpc(name = "rootUnsubscribe")] + fn root_unsubscribe(&self, id: SubscriptionId) -> Result; + } } pub struct RpcSolPubSubImpl { - uid: Arc, - subscriptions: Arc, - max_active_subscriptions: usize, + config: PubSubConfig, + subscription_control: SubscriptionControl, + current_subscriptions: Arc>, } impl RpcSolPubSubImpl { - pub fn new(subscriptions: Arc, max_active_subscriptions: usize) -> Self { - let uid = Arc::new(atomic::AtomicUsize::default()); + pub fn new( + config: PubSubConfig, + subscription_control: SubscriptionControl, + current_subscriptions: Arc>, + ) -> Self { Self { - uid, - subscriptions, - max_active_subscriptions, + config, + subscription_control, + current_subscriptions, } } - #[cfg(test)] - fn default_with_bank_forks(bank_forks: Arc>) -> Self { - let uid = Arc::new(atomic::AtomicUsize::default()); - let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); - let max_active_subscriptions = MAX_ACTIVE_SUBSCRIPTIONS; - Self { - uid, - subscriptions, - max_active_subscriptions, - } - } - - fn check_subscription_count(&self) -> Result<()> { - let num_subscriptions = self.subscriptions.total(); - debug!("Total existing subscriptions: {}", num_subscriptions); - if num_subscriptions >= self.max_active_subscriptions { - info!("Node subscription limit reached"); - datapoint_info!("rpc-subscription", ("total", num_subscriptions, i64)); - inc_new_counter_info!("rpc-subscription-refused-limit-reached", 1); - Err(Error { + fn subscribe(&self, params: SubscriptionParams) -> Result { + let token = self + .subscription_control + .subscribe(params) + .map_err(|_| Error { code: ErrorCode::InternalError, message: "Internal Error: Subscription refused. Node subscription limit reached" .into(), data: None, - }) + })?; + let id = token.id(); + self.current_subscriptions.insert(id, token); + Ok(id) + } + + fn unsubscribe(&self, id: SubscriptionId) -> Result { + if self.current_subscriptions.remove(&id).is_some() { + Ok(true) } else { - datapoint_info!("rpc-subscription", ("total", num_subscriptions + 1, i64)); - Ok(()) + Err(Error { + code: ErrorCode::InvalidParams, + message: "Invalid subscription id.".into(), + data: None, + }) } } } @@ -245,326 +367,149 @@ fn param(param_str: &str, thing: &str) -> Result { }) } -impl RpcSolPubSub for RpcSolPubSubImpl { - type Metadata = Arc; - +impl RpcSolPubSubInternal for RpcSolPubSubImpl { fn account_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, pubkey_str: String, config: Option, - ) { - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - match param::(&pubkey_str, "pubkey") { - Ok(pubkey) => { - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("account_subscribe: account={:?} id={:?}", pubkey, sub_id); - self.subscriptions - .add_account_subscription(pubkey, config, sub_id, subscriber) - } - Err(e) => subscriber.reject(e).unwrap_or_default(), - } + ) -> Result { + let config = config.unwrap_or_default(); + let params = AccountSubscriptionParams { + pubkey: param::(&pubkey_str, "pubkey")?, + commitment: config.commitment.unwrap_or_default(), + data_slice: config.data_slice, + encoding: config.encoding.unwrap_or(UiAccountEncoding::Binary), + }; + self.subscribe(SubscriptionParams::Account(params)) } - fn account_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("account_unsubscribe: id={:?}", id); - if self.subscriptions.remove_account_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn account_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } fn program_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, pubkey_str: String, config: Option, - ) { - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - match param::(&pubkey_str, "pubkey") { - Ok(pubkey) => { - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("program_subscribe: account={:?} id={:?}", pubkey, sub_id); - self.subscriptions - .add_program_subscription(pubkey, config, sub_id, subscriber) - } - Err(e) => subscriber.reject(e).unwrap_or_default(), - } + ) -> Result { + let config = config.unwrap_or_default(); + let params = ProgramSubscriptionParams { + pubkey: param::(&pubkey_str, "pubkey")?, + filters: config.filters.unwrap_or_default(), + encoding: config + .account_config + .encoding + .unwrap_or(UiAccountEncoding::Binary), + data_slice: config.account_config.data_slice, + commitment: config.account_config.commitment.unwrap_or_default(), + with_context: config.with_context.unwrap_or_default(), + }; + self.subscribe(SubscriptionParams::Program(params)) } - fn program_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("program_unsubscribe: id={:?}", id); - if self.subscriptions.remove_program_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn program_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } fn logs_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, filter: RpcTransactionLogsFilter, config: Option, - ) { - info!("logs_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - - let (address, include_votes) = match filter { - RpcTransactionLogsFilter::All => (None, false), - RpcTransactionLogsFilter::AllWithVotes => (None, true), - RpcTransactionLogsFilter::Mentions(addresses) => { - match addresses.len() { - 1 => match param::(&addresses[0], "mentions") { - Ok(address) => (Some(address), false), - Err(e) => { - subscriber.reject(e).unwrap_or_default(); - return; - } - }, - _ => { - // Room is reserved in the API to support multiple addresses, but for now - // the implementation only supports one - subscriber - .reject(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Only 1 address supported".into(), - data: None, - }) - .unwrap_or_default(); - return; + ) -> Result { + let params = LogsSubscriptionParams { + kind: match filter { + RpcTransactionLogsFilter::All => LogsSubscriptionKind::All, + RpcTransactionLogsFilter::AllWithVotes => LogsSubscriptionKind::AllWithVotes, + RpcTransactionLogsFilter::Mentions(keys) => { + if keys.len() != 1 { + return Err(Error { + code: ErrorCode::InvalidParams, + message: "Invalid Request: Only 1 address supported".into(), + data: None, + }); } + LogsSubscriptionKind::Single(param::(&keys[0], "mentions")?) } - } + }, + commitment: config.and_then(|c| c.commitment).unwrap_or_default(), }; - - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - self.subscriptions.add_logs_subscription( - address, - include_votes, - config.and_then(|config| config.commitment), - sub_id, - subscriber, - ) + self.subscribe(SubscriptionParams::Logs(params)) } - fn logs_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("logs_unsubscribe: id={:?}", id); - if self.subscriptions.remove_logs_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn logs_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } fn signature_subscribe( &self, - _meta: Self::Metadata, - subscriber: Subscriber>, signature_str: String, - signature_subscribe_config: Option, - ) { - info!("signature_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - match param::(&signature_str, "signature") { - Ok(signature) => { - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!( - "signature_subscribe: signature={:?} id={:?}", - signature, sub_id - ); - self.subscriptions.add_signature_subscription( - signature, - signature_subscribe_config, - sub_id, - subscriber, - ); - } - Err(e) => subscriber.reject(e).unwrap_or_default(), - } + config: Option, + ) -> Result { + let config = config.unwrap_or_default(); + let params = SignatureSubscriptionParams { + signature: param::(&signature_str, "signature")?, + commitment: config.commitment.unwrap_or_default(), + enable_received_notification: config.enable_received_notification.unwrap_or_default(), + }; + self.subscribe(SubscriptionParams::Signature(params)) } - fn signature_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("signature_unsubscribe"); - if self.subscriptions.remove_signature_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn signature_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } - fn slot_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber) { - info!("slot_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("slot_subscribe: id={:?}", sub_id); - self.subscriptions.add_slot_subscription(sub_id, subscriber); + fn slot_subscribe(&self) -> Result { + self.subscribe(SubscriptionParams::Slot) } - fn slot_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("slot_unsubscribe"); - if self.subscriptions.remove_slot_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn slot_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } - fn slots_updates_subscribe( - &self, - _meta: Self::Metadata, - subscriber: Subscriber>, - ) { - info!("slots_updates_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("slots_updates_subscribe: id={:?}", sub_id); - self.subscriptions - .add_slots_updates_subscription(sub_id, subscriber); + fn slots_updates_subscribe(&self) -> Result { + self.subscribe(SubscriptionParams::SlotsUpdates) } - fn slots_updates_unsubscribe( - &self, - _meta: Option, - id: SubscriptionId, - ) -> Result { - info!("slots_updates_unsubscribe"); - if self.subscriptions.remove_slots_updates_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn slots_updates_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } - fn vote_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber) { - info!("vote_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; + fn vote_subscribe(&self) -> Result { + if !self.config.enable_vote_subscription { + return Err(Error::new(jsonrpc_core::ErrorCode::MethodNotFound)); } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("vote_subscribe: id={:?}", sub_id); - self.subscriptions.add_vote_subscription(sub_id, subscriber); + self.subscribe(SubscriptionParams::Vote) } - fn vote_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("vote_unsubscribe"); - if self.subscriptions.remove_vote_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) + fn vote_unsubscribe(&self, id: SubscriptionId) -> Result { + if !self.config.enable_vote_subscription { + return Err(Error::new(jsonrpc_core::ErrorCode::MethodNotFound)); } + self.unsubscribe(id) } - fn root_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber) { - info!("root_subscribe"); - if let Err(err) = self.check_subscription_count() { - subscriber.reject(err).unwrap_or_default(); - return; - } - let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed); - let sub_id = SubscriptionId::Number(id as u64); - info!("root_subscribe: id={:?}", sub_id); - self.subscriptions.add_root_subscription(sub_id, subscriber); + fn root_subscribe(&self) -> Result { + self.subscribe(SubscriptionParams::Root) } - fn root_unsubscribe(&self, _meta: Option, id: SubscriptionId) -> Result { - info!("root_unsubscribe"); - if self.subscriptions.remove_root_subscription(&id) { - Ok(true) - } else { - Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid Request: Subscription id does not exist".into(), - data: None, - }) - } + fn root_unsubscribe(&self, id: SubscriptionId) -> Result { + self.unsubscribe(id) } } #[cfg(test)] mod tests { - use super::*; + use super::{RpcSolPubSubInternal, *}; use crate::{ - cluster_info_vote_listener::{ClusterInfoVoteListener, VoteTracker}, - optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, - rpc_subscriptions::tests::robust_poll_or_panic, + optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, rpc_pubsub_service, + rpc_subscriptions::RpcSubscriptions, }; - use crossbeam_channel::unbounded; - use jsonrpc_core::{futures::channel::mpsc, Response}; - use jsonrpc_pubsub::{PubSubHandler, Session}; + use jsonrpc_core::{IoHandler, Response}; use serial_test::serial; use solana_account_decoder::{parse_account_data::parse_account_data, UiAccountEncoding}; - use solana_client::rpc_response::{ProcessedSignatureResult, ReceivedSignatureResult}; + use solana_client::rpc_response::{ + ProcessedSignatureResult, ReceivedSignatureResult, RpcSignatureResult, SlotInfo, + }; use solana_runtime::{ bank::Bank, bank_forks::BankForks, @@ -576,6 +521,7 @@ mod tests { }; use solana_sdk::{ account::ReadableAccount, + clock::Slot, commitment_config::CommitmentConfig, hash::Hash, message::Message, @@ -588,7 +534,7 @@ mod tests { self, stake_instruction, stake_state::{Authorized, Lockup, StakeAuthorize, StakeState}, }; - use solana_vote_program::vote_transaction; + use solana_vote_program::vote_state::Vote; use std::{ sync::{atomic::AtomicBool, RwLock}, thread::sleep, @@ -615,10 +561,6 @@ mod tests { Ok(()) } - fn create_session() -> Arc { - Arc::new(Session::new(mpsc::unbounded().0)) - } - #[test] #[serial] fn test_signature_subscribe() { @@ -632,36 +574,31 @@ mod tests { let bank = Bank::new(&genesis_config); let blockhash = bank.last_blockhash(); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl { - subscriptions: Arc::new(RpcSubscriptions::new( - &Arc::new(AtomicBool::new(false)), - bank_forks.clone(), - Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), - OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - )), - uid: Arc::new(atomic::AtomicUsize::default()), - max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, - }; + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( + &Arc::new(AtomicBool::new(false)), + bank_forks.clone(), + Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), + )); // Test signature subscriptions let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification"); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.signature_subscribe( - session, - subscriber, tx.signatures[0].to_string(), Some(RpcSignatureSubscribeConfig { commitment: Some(CommitmentConfig::finalized()), ..RpcSignatureSubscribeConfig::default() }), - ); + ) + .unwrap(); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 0).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 0).unwrap(); // Test signature confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = RpcSignatureResult::ProcessedSignature(ProcessedSignatureResult { err: None }); let expected = json!({ @@ -676,26 +613,27 @@ mod tests { } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); // Test "received" - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification"); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.signature_subscribe( - session, - subscriber, tx.signatures[0].to_string(), Some(RpcSignatureSubscribeConfig { commitment: Some(CommitmentConfig::finalized()), enable_received_notification: Some(true), }), - ); + ) + .unwrap(); let received_slot = 1; - rpc.subscriptions - .notify_signatures_received((received_slot, vec![tx.signatures[0]])); + rpc_subscriptions.notify_signatures_received((received_slot, vec![tx.signatures[0]])); // Test signature confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = RpcSignatureResult::ReceivedSignature(ReceivedSignatureResult::ReceivedSignature); let expected = json!({ @@ -709,26 +647,27 @@ mod tests { "subscription": 1, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); // Test "received" for gossip subscription - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("signatureNotification"); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.signature_subscribe( - session, - subscriber, tx.signatures[0].to_string(), Some(RpcSignatureSubscribeConfig { commitment: Some(CommitmentConfig::confirmed()), enable_received_notification: Some(true), }), - ); + ) + .unwrap(); let received_slot = 2; - rpc.subscriptions - .notify_signatures_received((received_slot, vec![tx.signatures[0]])); + rpc_subscriptions.notify_signatures_received((received_slot, vec![tx.signatures[0]])); // Test signature confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = RpcSignatureResult::ReceivedSignature(ReceivedSignatureResult::ReceivedSignature); let expected = json!({ @@ -742,7 +681,10 @@ mod tests { "subscription": 2, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); } #[test] @@ -758,10 +700,10 @@ mod tests { let blockhash = bank.last_blockhash(); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let session = create_session(); + let mut io = IoHandler::<()>::default(); + let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, _receiver) = rpc_pubsub_service::test_connection(&subscriptions); - let mut io = PubSubHandler::default(); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); io.extend_with(rpc.to_delegate()); let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash); @@ -769,10 +711,10 @@ mod tests { r#"{{"jsonrpc":"2.0","id":1,"method":"signatureSubscribe","params":["{}"]}}"#, tx.signatures[0].to_string() ); - let _res = io.handle_request_sync(&req, session.clone()); + let _res = io.handle_request_sync(&req); let req = r#"{"jsonrpc":"2.0","id":1,"method":"signatureUnsubscribe","params":[0]}"#; - let res = io.handle_request_sync(&req, session.clone()); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","result":true,"id":1}"#; let expected: Response = serde_json::from_str(&expected).unwrap(); @@ -782,7 +724,7 @@ mod tests { // Test bad parameter let req = r#"{"jsonrpc":"2.0","id":1,"method":"signatureUnsubscribe","params":[1]}"#; - let res = io.handle_request_sync(&req, session); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid subscription id."},"id":1}"#; let expected: Response = serde_json::from_str(&expected).unwrap(); @@ -811,33 +753,35 @@ mod tests { let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); bank_forks.write().unwrap().insert(bank1); - let rpc = RpcSolPubSubImpl { - subscriptions: Arc::new(RpcSubscriptions::new( - &Arc::new(AtomicBool::new(false)), - bank_forks.clone(), - Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( - 1, 1, - ))), - OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - )), - uid: Arc::new(atomic::AtomicUsize::default()), - max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, - }; - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( + &Arc::new(AtomicBool::new(false)), + bank_forks.clone(), + Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( + 1, 1, + ))), + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), + )); + + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.account_subscribe( - session, - subscriber, stake_account.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::processed()), encoding: None, data_slice: None, }), - ); + ) + .unwrap(); + + // Make sure the subscription is processed before continuing. + let (rpc2, mut receiver2) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc2.slot_subscribe().unwrap(); + rpc_subscriptions.notify_slot(1, 0, 0); + receiver2.recv(); let tx = system_transaction::transfer(&alice, &from.pubkey(), 51, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); let authorized = Authorized::auto(&stake_authority.pubkey()); let ixs = stake_instruction::create_account( @@ -849,8 +793,7 @@ mod tests { ); let message = Message::new(&ixs, Some(&from.pubkey())); let tx = Transaction::new(&[&from, &stake_account], message, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); - sleep(Duration::from_millis(200)); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); // Test signature confirmation notification #1 let account = bank_forks @@ -879,11 +822,14 @@ mod tests { } }); - let (response, _) = robust_poll_or_panic(receiver); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + let response = receiver.recv(); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); let tx = system_transaction::transfer(&alice, &stake_authority.pubkey(), 1, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); sleep(Duration::from_millis(200)); let ix = stake_instruction::authorize( &stake_account.pubkey(), @@ -894,7 +840,7 @@ mod tests { ); let message = Message::new(&[ix], Some(&stake_authority.pubkey())); let tx = Transaction::new(&[&stake_authority], message, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); sleep(Duration::from_millis(200)); let bank = bank_forks.read().unwrap()[1].clone(); @@ -922,30 +868,32 @@ mod tests { let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); bank_forks.write().unwrap().insert(bank1); - let rpc = RpcSolPubSubImpl { - subscriptions: Arc::new(RpcSubscriptions::new( - &Arc::new(AtomicBool::new(false)), - bank_forks.clone(), - Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( - 1, 1, - ))), - OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - )), - uid: Arc::new(atomic::AtomicUsize::default()), - max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, - }; - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( + &Arc::new(AtomicBool::new(false)), + bank_forks.clone(), + Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( + 1, 1, + ))), + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), + )); + + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.account_subscribe( - session, - subscriber, nonce_account.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::processed()), encoding: Some(UiAccountEncoding::JsonParsed), data_slice: None, }), - ); + ) + .unwrap(); + + // Make sure the subscription is processed before continuing. + let (rpc2, mut receiver2) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc2.slot_subscribe().unwrap(); + rpc_subscriptions.notify_slot(1, 0, 0); + receiver2.recv(); let ixs = system_instruction::create_nonce_account( &alice.pubkey(), @@ -955,8 +903,7 @@ mod tests { ); let message = Message::new(&ixs, Some(&alice.pubkey())); let tx = Transaction::new(&[&alice, &nonce_account], message, blockhash); - process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions, 1).unwrap(); - sleep(Duration::from_millis(200)); + process_transaction_and_notify(&bank_forks, &tx, &rpc_subscriptions, 1).unwrap(); // Test signature confirmation notification #1 let account = bank_forks @@ -992,20 +939,24 @@ mod tests { } }); - let (response, _) = robust_poll_or_panic(receiver); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + let response = receiver.recv(); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); } #[test] #[serial] fn test_account_unsubscribe() { let bob_pubkey = solana_sdk::pubkey::new_rand(); - let session = create_session(); + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(&genesis_config)))); - let mut io = PubSubHandler::default(); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); + let mut io = IoHandler::<()>::default(); + let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, _receiver) = rpc_pubsub_service::test_connection(&subscriptions); io.extend_with(rpc.to_delegate()); @@ -1013,10 +964,10 @@ mod tests { r#"{{"jsonrpc":"2.0","id":1,"method":"accountSubscribe","params":["{}"]}}"#, bob_pubkey.to_string() ); - let _res = io.handle_request_sync(&req, session.clone()); + let _res = io.handle_request_sync(&req); let req = r#"{"jsonrpc":"2.0","id":1,"method":"accountUnsubscribe","params":[0]}"#; - let res = io.handle_request_sync(&req, session.clone()); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","result":true,"id":1}"#; let expected: Response = serde_json::from_str(&expected).unwrap(); @@ -1026,7 +977,7 @@ mod tests { // Test bad parameter let req = r#"{"jsonrpc":"2.0","id":1,"method":"accountUnsubscribe","params":[1]}"#; - let res = io.handle_request_sync(&req, session); + let res = io.handle_request_sync(req); let expected = r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid subscription id."},"id":1}"#; let expected: Response = serde_json::from_str(&expected).unwrap(); @@ -1047,27 +998,25 @@ mod tests { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let bob = Keypair::new(); - let mut rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks.clone()); let exit = Arc::new(AtomicBool::new(false)); - let subscriptions = RpcSubscriptions::new( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - ); - rpc.subscriptions = Arc::new(subscriptions); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + )); + + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.account_subscribe( - session, - subscriber, bob.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::finalized()), encoding: None, data_slice: None, }), - ); + ) + .unwrap(); let tx = system_transaction::transfer(&alice, &bob.pubkey(), 100, blockhash); bank_forks @@ -1077,11 +1026,11 @@ mod tests { .unwrap() .process_transaction(&tx) .unwrap(); - rpc.subscriptions - .notify_subscribers(CommitmentSlots::default()); + rpc_subscriptions.notify_subscribers(CommitmentSlots::default()); + // allow 200ms for notification thread to wake std::thread::sleep(Duration::from_millis(200)); - let _panic = robust_poll_or_panic(receiver); + let _panic = receiver.recv(); } #[test] @@ -1099,29 +1048,26 @@ mod tests { bank_forks.write().unwrap().insert(bank1); let bob = Keypair::new(); - let mut rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks.clone()); let exit = Arc::new(AtomicBool::new(false)); let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache, OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - ); - rpc.subscriptions = Arc::new(subscriptions); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification"); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + rpc.account_subscribe( - session, - subscriber, bob.pubkey().to_string(), Some(RpcAccountInfoConfig { commitment: Some(CommitmentConfig::finalized()), encoding: None, data_slice: None, }), - ); + ) + .unwrap(); let tx = system_transaction::transfer(&alice, &bob.pubkey(), 100, blockhash); bank_forks @@ -1135,7 +1081,7 @@ mod tests { slot: 1, ..CommitmentSlots::default() }; - rpc.subscriptions.notify_subscribers(commitment_slots); + subscriptions.notify_subscribers(commitment_slots); let commitment_slots = CommitmentSlots { slot: 2, @@ -1143,7 +1089,7 @@ mod tests { highest_confirmed_slot: 1, highest_confirmed_root: 1, }; - rpc.subscriptions.notify_subscribers(commitment_slots); + subscriptions.notify_subscribers(commitment_slots); let expected = json!({ "jsonrpc": "2.0", "method": "accountNotification", @@ -1161,8 +1107,11 @@ mod tests { "subscription": 0, } }); - let (response, _) = robust_poll_or_panic(receiver); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + let response = receiver.recv(); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); } #[test] @@ -1171,21 +1120,21 @@ mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification"); - rpc.slot_subscribe(session, subscriber); + let rpc_subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + rpc.slot_subscribe().unwrap(); + + rpc_subscriptions.notify_slot(0, 0, 0); - rpc.subscriptions.notify_slot(0, 0, 0); // Test slot confirmation notification - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); let expected_res = SlotInfo { parent: 0, slot: 0, root: 0, }; - let expected_res_str = - serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap(); + let expected_res_str = serde_json::to_string(&expected_res).unwrap(); + let expected = format!( r#"{{"jsonrpc":"2.0","method":"slotNotification","params":{{"result":{},"subscription":0}}}}"#, expected_res_str @@ -1199,34 +1148,27 @@ mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification"); - rpc.slot_subscribe(session, subscriber); - rpc.subscriptions.notify_slot(0, 0, 0); - let (response, _) = robust_poll_or_panic(receiver); + let rpc_subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + let sub_id = rpc.slot_subscribe().unwrap(); + + rpc_subscriptions.notify_slot(0, 0, 0); + let response = receiver.recv(); let expected_res = SlotInfo { parent: 0, slot: 0, root: 0, }; - let expected_res_str = - serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap(); + let expected_res_str = serde_json::to_string(&expected_res).unwrap(); + let expected = format!( r#"{{"jsonrpc":"2.0","method":"slotNotification","params":{{"result":{},"subscription":0}}}}"#, expected_res_str ); assert_eq!(expected, response); - let session = create_session(); - assert!(rpc - .slot_unsubscribe(Some(session), SubscriptionId::Number(42)) - .is_err()); - - let session = create_session(); - assert!(rpc - .slot_unsubscribe(Some(session), SubscriptionId::Number(0)) - .is_ok()); + assert!(rpc.slot_unsubscribe(42.into()).is_err()); + assert!(rpc.slot_unsubscribe(sub_id).is_ok()); } #[test] @@ -1243,69 +1185,32 @@ mod tests { ); let exit = Arc::new(AtomicBool::new(false)); let bank = Bank::new(&genesis_config); - let bank_forks = BankForks::new(bank); - let bank = bank_forks.get(0).unwrap().clone(); - let bank_forks = Arc::new(RwLock::new(bank_forks)); - - // Setup RPC - let mut rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks.clone()); - let session = create_session(); - let (subscriber, _id_receiver, receiver) = Subscriber::new_test("voteNotification"); + let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); // Setup Subscriptions let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new_with_vote_subscription( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, block_commitment_cache, optimistically_confirmed_bank, - true, - ); - rpc.subscriptions = Arc::new(subscriptions); - rpc.vote_subscribe(session, subscriber); + )); + // Setup RPC + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + rpc.vote_subscribe().unwrap(); - // Create some voters at genesis - let vote_tracker = VoteTracker::new(&bank); - let (votes_sender, votes_receiver) = unbounded(); - let (vote_tracker, validator_voting_keypairs) = - (Arc::new(vote_tracker), validator_voting_keypairs); + let vote = Vote { + slots: vec![1, 2], + hash: Hash::default(), + timestamp: None, + }; + subscriptions.notify_vote(&vote); - let vote_slots = vec![1, 2]; - validator_voting_keypairs.iter().for_each(|keypairs| { - let node_keypair = &keypairs.node_keypair; - let vote_keypair = &keypairs.vote_keypair; - let vote_tx = vote_transaction::new_vote_transaction( - vote_slots.clone(), - Hash::default(), - Hash::default(), - node_keypair, - vote_keypair, - vote_keypair, - None, - ); - votes_sender.send(vec![vote_tx]).unwrap(); - }); - - // Process votes and check they were notified. - let (verified_vote_sender, _verified_vote_receiver) = unbounded(); - let (gossip_verified_vote_hash_sender, _gossip_verified_vote_hash_receiver) = unbounded(); - let (_replay_votes_sender, replay_votes_receiver) = unbounded(); - ClusterInfoVoteListener::get_and_process_votes_for_tests( - &votes_receiver, - &vote_tracker, - &bank, - &rpc.subscriptions, - &gossip_verified_vote_hash_sender, - &verified_vote_sender, - &replay_votes_receiver, - ) - .unwrap(); - - let (response, _) = robust_poll_or_panic(receiver); + let response = receiver.recv(); assert_eq!( response, - r#"{"jsonrpc":"2.0","method":"voteNotification","params":{"result":{"hash":"11111111111111111111111111111111","slots":[1,2],"timestamp":null},"subscription":0}}"# + r#"{"jsonrpc":"2.0","method":"voteNotification","params":{"result":{"slots":[1,2],"hash":"11111111111111111111111111111111","timestamp":null},"subscription":0}}"# ); } @@ -1315,19 +1220,11 @@ mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let rpc = RpcSolPubSubImpl::default_with_bank_forks(bank_forks); - let session = create_session(); - let (subscriber, _id_receiver, _) = Subscriber::new_test("voteNotification"); - rpc.vote_subscribe(session, subscriber); + let rpc_subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + let (rpc, _receiver) = rpc_pubsub_service::test_connection(&rpc_subscriptions); + let sub_id = rpc.vote_subscribe().unwrap(); - let session = create_session(); - assert!(rpc - .vote_unsubscribe(Some(session), SubscriptionId::Number(42)) - .is_err()); - - let session = create_session(); - assert!(rpc - .vote_unsubscribe(Some(session), SubscriptionId::Number(0)) - .is_ok()); + assert!(rpc.vote_unsubscribe(42.into()).is_err()); + assert!(rpc.vote_unsubscribe(sub_id).is_ok()); } } diff --git a/core/src/rpc_pubsub_service.rs b/core/src/rpc_pubsub_service.rs index 8f11997fa0..981a3c5425 100644 --- a/core/src/rpc_pubsub_service.rs +++ b/core/src/rpc_pubsub_service.rs @@ -1,44 +1,59 @@ //! The `pubsub` module implements a threaded subscription service on client RPC request use crate::{ - rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl, MAX_ACTIVE_SUBSCRIPTIONS}, - rpc_subscriptions::RpcSubscriptions, -}; -use jsonrpc_pubsub::{PubSubHandler, Session}; -use jsonrpc_ws_server::{RequestContext, ServerBuilder}; -use std::{ - net::SocketAddr, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, + rpc_pubsub::{RpcSolPubSubImpl, RpcSolPubSubInternal}, + rpc_subscription_tracker::{ + SubscriptionControl, SubscriptionId, SubscriptionParams, SubscriptionToken, }, - thread::{self, sleep, Builder, JoinHandle}, - time::Duration, + rpc_subscriptions::{RpcNotification, RpcSubscriptions}, }; +use dashmap::{mapref::entry::Entry, DashMap}; +use jsonrpc_core::IoHandler; +use soketto::handshake::{server, Server}; +use solana_metrics::TokenCounter; +use std::{ + io, + net::SocketAddr, + str, + sync::Arc, + thread::{self, Builder, JoinHandle}, +}; +use stream_cancel::{Trigger, Tripwire}; +use thiserror::Error; +use tokio::{net::TcpStream, pin, select, sync::broadcast}; +use tokio_util::compat::TokioAsyncReadCompatExt; + +pub const MAX_ACTIVE_SUBSCRIPTIONS: usize = 1_000_000; +pub const DEFAULT_QUEUE_CAPACITY_ITEMS: usize = 10_000_000; +pub const DEFAULT_TEST_QUEUE_CAPACITY_ITEMS: usize = 100; +pub const DEFAULT_QUEUE_CAPACITY_BYTES: usize = 256 * 1024 * 1024; #[derive(Debug, Clone)] pub struct PubSubConfig { pub enable_vote_subscription: bool, - - // See the corresponding fields in - // https://github.com/paritytech/ws-rs/blob/be4d47575bae55c60d9f51b47480d355492a94fc/src/lib.rs#L131 - // for a complete description of each field in this struct - pub max_connections: usize, - pub max_fragment_size: usize, - pub max_in_buffer_capacity: usize, - pub max_out_buffer_capacity: usize, pub max_active_subscriptions: usize, + pub queue_capacity_items: usize, + pub queue_capacity_bytes: usize, } impl Default for PubSubConfig { fn default() -> Self { Self { enable_vote_subscription: false, - max_connections: 1000, // Arbitrary, default of 100 is too low - max_fragment_size: 50 * 1024, // 50KB - max_in_buffer_capacity: 50 * 1024, // 50KB - max_out_buffer_capacity: 15 * 1024 * 1024, // max account size (10MB), then 5MB extra for base64 encoding overhead/etc max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, + queue_capacity_items: DEFAULT_QUEUE_CAPACITY_ITEMS, + queue_capacity_bytes: DEFAULT_QUEUE_CAPACITY_BYTES, + } + } +} + +impl PubSubConfig { + pub fn default_for_tests() -> Self { + Self { + enable_vote_subscription: false, + max_active_subscriptions: MAX_ACTIVE_SUBSCRIPTIONS, + queue_capacity_items: DEFAULT_TEST_QUEUE_CAPACITY_ITEMS, + queue_capacity_bytes: DEFAULT_QUEUE_CAPACITY_BYTES, } } } @@ -52,51 +67,30 @@ impl PubSubService { pubsub_config: PubSubConfig, subscriptions: &Arc, pubsub_addr: SocketAddr, - exit: &Arc, - ) -> Self { + ) -> (Trigger, Self) { + let subscription_control = subscriptions.control().clone(); info!("rpc_pubsub bound to {:?}", pubsub_addr); - let rpc = RpcSolPubSubImpl::new( - subscriptions.clone(), - pubsub_config.max_active_subscriptions, - ); - let exit_ = exit.clone(); + let (trigger, tripwire) = Tripwire::new(); let thread_hdl = Builder::new() .name("solana-pubsub".to_string()) .spawn(move || { - let mut io = PubSubHandler::default(); - io.extend_with(rpc.to_delegate()); - - let server = ServerBuilder::with_meta_extractor(io, |context: &RequestContext| { - info!("New pubsub connection"); - let session = Arc::new(Session::new(context.sender())); - session.on_drop(|| { - info!("Pubsub connection dropped"); - }); - session - }) - .max_connections(pubsub_config.max_connections) - .max_payload(pubsub_config.max_fragment_size) - .max_in_buffer_capacity(pubsub_config.max_in_buffer_capacity) - .max_out_buffer_capacity(pubsub_config.max_out_buffer_capacity) - .start(&pubsub_addr); - - if let Err(e) = server { - warn!( - "Pubsub service unavailable error: {:?}. \n\ - Also, check that port {} is not already in use by another application", - e, - pubsub_addr.port() - ); - return; - } - while !exit_.load(Ordering::Relaxed) { - sleep(Duration::from_millis(100)); - } - server.unwrap().close(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime creation failed"); + if let Err(err) = runtime.block_on(listen( + pubsub_addr, + pubsub_config, + subscription_control, + tripwire, + )) { + error!("pubsub service failed: {}", err); + }; }) - .unwrap(); - Self { thread_hdl } + .expect("thread spawn failed"); + + (trigger, Self { thread_hdl }) } pub fn close(self) -> thread::Result<()> { @@ -108,6 +102,244 @@ impl PubSubService { } } +struct BroadcastHandler { + current_subscriptions: Arc>, +} + +fn count_final(params: &SubscriptionParams) { + match params { + SubscriptionParams::Account(_) => { + inc_new_counter_info!("rpc-pubsub-final-accounts", 1); + } + SubscriptionParams::Logs(_) => { + inc_new_counter_info!("rpc-pubsub-final-logs", 1); + } + SubscriptionParams::Program(_) => { + inc_new_counter_info!("rpc-pubsub-final-programs", 1); + } + SubscriptionParams::Signature(_) => { + inc_new_counter_info!("rpc-pubsub-final-signatures", 1); + } + SubscriptionParams::Slot => { + inc_new_counter_info!("rpc-pubsub-final-slots", 1); + } + SubscriptionParams::SlotsUpdates => { + inc_new_counter_info!("rpc-pubsub-final-slots-updates", 1); + } + SubscriptionParams::Root => { + inc_new_counter_info!("rpc-pubsub-final-roots", 1); + } + SubscriptionParams::Vote => { + inc_new_counter_info!("rpc-pubsub-final-votes", 1); + } + } +} + +impl BroadcastHandler { + fn handle(&self, notification: RpcNotification) -> Result>, Error> { + if let Entry::Occupied(entry) = self + .current_subscriptions + .entry(notification.subscription_id) + { + count_final(entry.get().params()); + + if notification.is_final { + entry.remove(); + } + notification + .json + .upgrade() + .ok_or(Error::NotificationIsGone) + .map(Some) + } else { + Ok(None) + } + } +} + +#[cfg(test)] +pub struct TestBroadcastReceiver { + handler: BroadcastHandler, + inner: tokio::sync::broadcast::Receiver, +} + +#[cfg(test)] +impl TestBroadcastReceiver { + pub fn recv(&mut self) -> String { + use std::thread::sleep; + use std::time::{Duration, Instant}; + use tokio::sync::broadcast::error::TryRecvError; + + let timeout = Duration::from_millis(500); + let started = Instant::now(); + + loop { + match self.inner.try_recv() { + Ok(notification) => { + if let Some(json) = self.handler.handle(notification).expect("handler failed") { + return json.to_string(); + } + } + Err(TryRecvError::Empty) => { + if started.elapsed() > timeout { + panic!("TestBroadcastReceiver: no data, timeout reached"); + } + sleep(Duration::from_millis(50)); + } + Err(err) => panic!("broadcast receiver error: {}", err), + } + } + } +} + +#[cfg(test)] +pub fn test_connection( + subscriptions: &Arc, +) -> (RpcSolPubSubImpl, TestBroadcastReceiver) { + let current_subscriptions = Arc::new(DashMap::new()); + + let rpc_impl = RpcSolPubSubImpl::new( + PubSubConfig { + enable_vote_subscription: true, + queue_capacity_items: 100, + ..PubSubConfig::default() + }, + subscriptions.control().clone(), + Arc::clone(¤t_subscriptions), + ); + let broadcast_handler = BroadcastHandler { + current_subscriptions, + }; + let receiver = TestBroadcastReceiver { + inner: subscriptions.control().broadcast_receiver(), + handler: broadcast_handler, + }; + (rpc_impl, receiver) +} + +#[derive(Debug, Error)] +enum Error { + #[error("handshake error: {0}")] + Handshake(#[from] soketto::handshake::Error), + #[error("connection error: {0}")] + Connection(#[from] soketto::connection::Error), + #[error("broadcast queue error: {0}")] + Broadcast(#[from] broadcast::error::RecvError), + #[error("client has lagged behind (notification is gone)")] + NotificationIsGone, +} + +async fn handle_connection( + socket: TcpStream, + subscription_control: SubscriptionControl, + config: PubSubConfig, + mut tripwire: Tripwire, +) -> Result<(), Error> { + let mut server = Server::new(socket.compat()); + let request = server.receive_request().await?; + let accept = server::Response::Accept { + key: request.key(), + protocol: None, + }; + server.send_response(&accept).await?; + let (mut sender, mut receiver) = server.into_builder().finish(); + + let mut broadcast_receiver = subscription_control.broadcast_receiver(); + let mut data = Vec::new(); + let current_subscriptions = Arc::new(DashMap::new()); + + let mut json_rpc_handler = IoHandler::new(); + let rpc_impl = RpcSolPubSubImpl::new( + config, + subscription_control, + Arc::clone(¤t_subscriptions), + ); + json_rpc_handler.extend_with(rpc_impl.to_delegate()); + let broadcast_handler = BroadcastHandler { + current_subscriptions, + }; + loop { + // Extra block for dropping `receive_future`. + { + // soketto is not cancel safe, so we have to introduce an inner loop to poll + // `receive_data` to completion. + let receive_future = receiver.receive_data(&mut data); + pin!(receive_future); + loop { + select! { + result = &mut receive_future => match result { + Ok(_) => break, + Err(soketto::connection::Error::Closed) => return Ok(()), + Err(err) => return Err(err.into()), + }, + result = broadcast_receiver.recv() => { + + // In both possible error cases (closed or lagged) we disconnect the client. + if let Some(json) = broadcast_handler.handle(result?)? { + sender.send_text(&*json).await?; + } + }, + _ = &mut tripwire => { + warn!("disconnecting websocket client: shutting down"); + return Ok(()) + }, + + } + } + } + let data_str = match str::from_utf8(&data) { + Ok(str) => str, + Err(_) => { + // Old implementation just closes the connection, so we preserve that behavior + // for now. It would be more correct to respond with an error. + break; + } + }; + + if let Some(response) = json_rpc_handler.handle_request(data_str).await { + sender.send_text(&response).await?; + } + data.clear(); + } + + Ok(()) +} + +async fn listen( + listen_address: SocketAddr, + config: PubSubConfig, + subscription_control: SubscriptionControl, + mut tripwire: Tripwire, +) -> io::Result<()> { + let listener = tokio::net::TcpListener::bind(&listen_address).await?; + let counter = TokenCounter::new("rpc_pubsub_connections"); + loop { + select! { + result = listener.accept() => match result { + Ok((socket, addr)) => { + debug!("new client ({:?})", addr); + let subscription_control = subscription_control.clone(); + let config = config.clone(); + let tripwire = tripwire.clone(); + let counter_token = counter.create_token(); + tokio::spawn(async move { + let handle = handle_connection( + socket, subscription_control, config, tripwire + ); + match handle.await { + Ok(()) => debug!("connection closed ({:?})", addr), + Err(err) => warn!("connection handler error ({:?}): {}", addr, err), + } + drop(counter_token); // Force moving token into the task. + }); + } + Err(e) => error!("couldn't accept connection: {:?}", e), + }, + _ = &mut tripwire => return Ok(()), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -120,7 +352,7 @@ mod tests { }; use std::{ net::{IpAddr, Ipv4Addr}, - sync::RwLock, + sync::{atomic::AtomicBool, RwLock}, }; #[test] @@ -132,14 +364,14 @@ mod tests { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, )); - let pubsub_service = - PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr, &exit); + let (_trigger, pubsub_service) = + PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr); let thread = pubsub_service.thread_hdl.thread(); assert_eq!(thread.name().unwrap(), "solana-pubsub"); } diff --git a/core/src/rpc_subscription_tracker.rs b/core/src/rpc_subscription_tracker.rs new file mode 100644 index 0000000000..45b4a03a5d --- /dev/null +++ b/core/src/rpc_subscription_tracker.rs @@ -0,0 +1,731 @@ +use { + crate::rpc_subscriptions::{NotificationEntry, RpcNotification}, + dashmap::{mapref::entry::Entry as DashEntry, DashMap}, + solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig}, + solana_client::rpc_filter::RpcFilterType, + solana_metrics::{CounterToken, TokenCounter}, + solana_runtime::{ + bank::{TransactionLogCollectorConfig, TransactionLogCollectorFilter}, + bank_forks::BankForks, + }, + solana_sdk::{ + clock::Slot, commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature, + }, + std::{ + collections::{ + hash_map::{Entry, HashMap}, + HashSet, + }, + fmt, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, RwLock, Weak, + }, + }, + thiserror::Error, + tokio::sync::broadcast, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct SubscriptionId(u64); + +impl From for SubscriptionId { + fn from(value: u64) -> Self { + SubscriptionId(value) + } +} + +impl From for u64 { + fn from(value: SubscriptionId) -> Self { + value.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SubscriptionParams { + Account(AccountSubscriptionParams), + Logs(LogsSubscriptionParams), + Program(ProgramSubscriptionParams), + Signature(SignatureSubscriptionParams), + Slot, + SlotsUpdates, + Root, + Vote, +} + +impl SubscriptionParams { + fn method(&self) -> &'static str { + match self { + SubscriptionParams::Account(_) => "accountNotification", + SubscriptionParams::Logs(_) => "logsNotification", + SubscriptionParams::Program(_) => "programNotification", + SubscriptionParams::Signature(_) => "signatureNotification", + SubscriptionParams::Slot => "slotNotification", + SubscriptionParams::SlotsUpdates => "slotsUpdatesNotification", + SubscriptionParams::Root => "rootNotification", + SubscriptionParams::Vote => "voteNotification", + } + } + + fn commitment(&self) -> Option { + match self { + SubscriptionParams::Account(params) => Some(params.commitment), + SubscriptionParams::Logs(params) => Some(params.commitment), + SubscriptionParams::Program(params) => Some(params.commitment), + SubscriptionParams::Signature(params) => Some(params.commitment), + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => None, + } + } + + fn is_commitment_watcher(&self) -> bool { + let commitment = match self { + SubscriptionParams::Account(params) => ¶ms.commitment, + SubscriptionParams::Logs(params) => ¶ms.commitment, + SubscriptionParams::Program(params) => ¶ms.commitment, + SubscriptionParams::Signature(params) => ¶ms.commitment, + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => return false, + }; + !commitment.is_confirmed() + } + + fn is_gossip_watcher(&self) -> bool { + let commitment = match self { + SubscriptionParams::Account(params) => ¶ms.commitment, + SubscriptionParams::Logs(params) => ¶ms.commitment, + SubscriptionParams::Program(params) => ¶ms.commitment, + SubscriptionParams::Signature(params) => ¶ms.commitment, + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => return false, + }; + commitment.is_confirmed() + } + + fn is_node_progress_watcher(&self) -> bool { + matches!( + self, + SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AccountSubscriptionParams { + pub pubkey: Pubkey, + pub encoding: UiAccountEncoding, + pub data_slice: Option, + pub commitment: CommitmentConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct LogsSubscriptionParams { + pub kind: LogsSubscriptionKind, + pub commitment: CommitmentConfig, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum LogsSubscriptionKind { + All, + AllWithVotes, + Single(Pubkey), +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ProgramSubscriptionParams { + pub pubkey: Pubkey, + pub filters: Vec, + pub encoding: UiAccountEncoding, + pub data_slice: Option, + pub commitment: CommitmentConfig, + pub with_context: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SignatureSubscriptionParams { + pub signature: Signature, + pub commitment: CommitmentConfig, + pub enable_received_notification: bool, +} + +#[derive(Clone)] +pub struct SubscriptionControl(Arc); + +struct SubscriptionControlInner { + subscriptions: DashMap>, + next_id: AtomicU64, + max_active_subscriptions: usize, + sender: crossbeam_channel::Sender, + broadcast_sender: broadcast::Sender, + counter: TokenCounter, +} + +impl SubscriptionControl { + pub fn new( + max_active_subscriptions: usize, + sender: crossbeam_channel::Sender, + broadcast_sender: broadcast::Sender, + ) -> Self { + Self(Arc::new(SubscriptionControlInner { + subscriptions: DashMap::new(), + next_id: AtomicU64::new(0), + max_active_subscriptions, + sender, + broadcast_sender, + counter: TokenCounter::new("rpc_pubsub_total_subscriptions"), + })) + } + + pub fn broadcast_receiver(&self) -> broadcast::Receiver { + self.0.broadcast_sender.subscribe() + } + + pub fn subscribe(&self, params: SubscriptionParams) -> Result { + debug!( + "Total existing subscriptions: {}", + self.0.subscriptions.len() + ); + let count = self.0.subscriptions.len(); + match self.0.subscriptions.entry(params) { + DashEntry::Occupied(entry) => Ok(SubscriptionToken( + entry + .get() + .upgrade() + .expect("dead subscription encountered in SubscriptionControl"), + self.0.counter.create_token(), + )), + DashEntry::Vacant(entry) => { + if count >= self.0.max_active_subscriptions { + inc_new_counter_info!("rpc-subscription-refused-limit-reached", 1); + return Err(Error::TooManySubscriptions); + } + let id = SubscriptionId::from(self.0.next_id.fetch_add(1, Ordering::AcqRel)); + let token = SubscriptionToken( + Arc::new(SubscriptionTokenInner { + control: Arc::clone(&self.0), + params: entry.key().clone(), + id, + }), + self.0.counter.create_token(), + ); + let _ = self + .0 + .sender + .send(NotificationEntry::Subscribed(token.0.params.clone(), id)); + entry.insert(Arc::downgrade(&token.0)); + datapoint_info!( + "rpc-subscription", + ("total", self.0.subscriptions.len(), i64) + ); + Ok(token) + } + } + } + + pub fn total(&self) -> usize { + self.0.subscriptions.len() + } + + #[cfg(test)] + pub fn assert_subscribed(&self, params: &SubscriptionParams) { + assert!(self.0.subscriptions.contains_key(params)); + } + + #[cfg(test)] + pub fn assert_unsubscribed(&self, params: &SubscriptionParams) { + assert!(!self.0.subscriptions.contains_key(params)); + } + + #[cfg(test)] + pub fn account_subscribed(&self, pubkey: &Pubkey) -> bool { + self.0.subscriptions.iter().any(|item| { + if let SubscriptionParams::Account(params) = item.key() { + ¶ms.pubkey == pubkey + } else { + false + } + }) + } + + #[cfg(test)] + pub fn signature_subscribed(&self, signature: &Signature) -> bool { + self.0.subscriptions.iter().any(|item| { + if let SubscriptionParams::Signature(params) = item.key() { + ¶ms.signature == signature + } else { + false + } + }) + } +} + +#[derive(Debug)] +pub struct SubscriptionInfo { + id: SubscriptionId, + params: SubscriptionParams, + method: &'static str, + pub last_notified_slot: RwLock, + commitment: Option, +} + +impl SubscriptionInfo { + pub fn id(&self) -> SubscriptionId { + self.id + } + + pub fn method(&self) -> &'static str { + self.method + } + + pub fn params(&self) -> &SubscriptionParams { + &self.params + } + + pub fn commitment(&self) -> Option { + self.commitment + } +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("node subscription limit reached")] + TooManySubscriptions, +} + +struct LogsSubscriptionsIndex { + all_count: usize, + all_with_votes_count: usize, + single_count: HashMap, + + bank_forks: Arc>, +} + +impl LogsSubscriptionsIndex { + fn add(&mut self, params: &LogsSubscriptionParams) { + match params.kind { + LogsSubscriptionKind::All => self.all_count += 1, + LogsSubscriptionKind::AllWithVotes => self.all_with_votes_count += 1, + LogsSubscriptionKind::Single(key) => { + *self.single_count.entry(key).or_default() += 1; + } + } + self.update_config(); + } + + fn remove(&mut self, params: &LogsSubscriptionParams) { + match params.kind { + LogsSubscriptionKind::All => self.all_count -= 1, + LogsSubscriptionKind::AllWithVotes => self.all_with_votes_count -= 1, + LogsSubscriptionKind::Single(key) => match self.single_count.entry(key) { + Entry::Occupied(mut entry) => { + *entry.get_mut() -= 1; + if *entry.get() == 0 { + entry.remove(); + } + } + Entry::Vacant(_) => error!("missing entry in single_count"), + }, + } + self.update_config(); + } + + fn update_config(&self) { + let config = if self.all_with_votes_count > 0 { + TransactionLogCollectorConfig { + filter: TransactionLogCollectorFilter::AllWithVotes, + mentioned_addresses: HashSet::new(), + } + } else if self.all_count > 0 { + TransactionLogCollectorConfig { + filter: TransactionLogCollectorFilter::All, + mentioned_addresses: HashSet::new(), + } + } else { + TransactionLogCollectorConfig { + filter: TransactionLogCollectorFilter::OnlyMentionedAddresses, + mentioned_addresses: self.single_count.keys().copied().collect(), + } + }; + + *self + .bank_forks + .read() + .unwrap() + .root_bank() + .transaction_log_collector_config + .write() + .unwrap() = config; + } +} + +pub struct SubscriptionsTracker { + logs_subscriptions_index: LogsSubscriptionsIndex, + by_signature: HashMap>>, + // Accounts, logs, programs, signatures (not gossip) + commitment_watchers: HashMap>, + // Accounts, logs, programs, signatures (gossip) + gossip_watchers: HashMap>, + // Slots, slots updates, roots, votes. + node_progress_watchers: HashMap>, +} + +impl SubscriptionsTracker { + pub fn new(bank_forks: Arc>) -> Self { + SubscriptionsTracker { + logs_subscriptions_index: LogsSubscriptionsIndex { + all_count: 0, + all_with_votes_count: 0, + single_count: HashMap::new(), + bank_forks, + }, + by_signature: HashMap::new(), + commitment_watchers: HashMap::new(), + gossip_watchers: HashMap::new(), + node_progress_watchers: HashMap::new(), + } + } + + pub fn subscribe( + &mut self, + params: SubscriptionParams, + id: SubscriptionId, + last_notified_slot: impl FnOnce() -> Slot, + ) { + let info = Arc::new(SubscriptionInfo { + last_notified_slot: RwLock::new(last_notified_slot()), + id, + commitment: params.commitment(), + method: params.method(), + params: params.clone(), + }); + match ¶ms { + SubscriptionParams::Logs(params) => { + self.logs_subscriptions_index.add(params); + } + SubscriptionParams::Signature(params) => { + self.by_signature + .entry(params.signature) + .or_default() + .insert(id, Arc::clone(&info)); + } + _ => {} + } + if info.params.is_commitment_watcher() { + self.commitment_watchers.insert(id, Arc::clone(&info)); + } + if info.params.is_gossip_watcher() { + self.gossip_watchers.insert(id, Arc::clone(&info)); + } + if info.params.is_node_progress_watcher() { + self.node_progress_watchers + .insert(info.params.clone(), Arc::clone(&info)); + } + } + + #[allow(clippy::collapsible_if)] + pub fn unsubscribe(&mut self, params: SubscriptionParams, id: SubscriptionId) { + match ¶ms { + SubscriptionParams::Logs(params) => { + self.logs_subscriptions_index.remove(params); + } + SubscriptionParams::Signature(params) => { + if let Entry::Occupied(mut entry) = self.by_signature.entry(params.signature) { + if entry.get_mut().remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in by_signature)"); + } + if entry.get_mut().is_empty() { + entry.remove(); + } + } else { + warn!("Subscriptions inconsistency (missing entry in by_signature)"); + } + } + _ => {} + } + if params.is_commitment_watcher() { + if self.commitment_watchers.remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in commitment_watchers)"); + } + } + if params.is_gossip_watcher() { + if self.gossip_watchers.remove(&id).is_none() { + warn!("Subscriptions inconsistency (missing entry in gossip_watchers)"); + } + } + if params.is_node_progress_watcher() { + if self.node_progress_watchers.remove(¶ms).is_none() { + warn!("Subscriptions inconsistency (missing entry in node_progress_watchers)"); + } + } + } + + pub fn by_signature( + &self, + ) -> &HashMap>> { + &self.by_signature + } + pub fn commitment_watchers(&self) -> &HashMap> { + &self.commitment_watchers + } + pub fn gossip_watchers(&self) -> &HashMap> { + &self.gossip_watchers + } + pub fn node_progress_watchers(&self) -> &HashMap> { + &self.node_progress_watchers + } +} + +struct SubscriptionTokenInner { + control: Arc, + params: SubscriptionParams, + id: SubscriptionId, +} + +impl fmt::Debug for SubscriptionTokenInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SubscriptionTokenInner") + .field("id", &self.id) + .finish() + } +} + +impl Drop for SubscriptionTokenInner { + #[allow(clippy::collapsible_if)] + fn drop(&mut self) { + match self.control.subscriptions.entry(self.params.clone()) { + DashEntry::Vacant(_) => { + warn!("Subscriptions inconsistency (missing entry in by_params)"); + } + DashEntry::Occupied(entry) => { + let _ = self.control.sender.send(NotificationEntry::Unsubscribed( + self.params.clone(), + self.id, + )); + entry.remove(); + datapoint_info!( + "rpc-subscription", + ("total", self.control.subscriptions.len(), i64) + ); + } + } + } +} + +#[derive(Clone)] +pub struct SubscriptionToken(Arc, CounterToken); + +impl SubscriptionToken { + pub fn id(&self) -> SubscriptionId { + self.0.id + } + + pub fn params(&self) -> &SubscriptionParams { + &self.0.params + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rpc_pubsub_service::PubSubConfig; + use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo}; + use solana_runtime::bank::Bank; + use std::str::FromStr; + + struct ControlWrapper { + control: SubscriptionControl, + receiver: crossbeam_channel::Receiver, + } + + impl ControlWrapper { + fn new() -> Self { + let (sender, receiver) = crossbeam_channel::unbounded(); + let (broadcast_sender, _broadcast_receiver) = broadcast::channel(42); + + let control = SubscriptionControl::new( + PubSubConfig::default().max_active_subscriptions, + sender, + broadcast_sender, + ); + Self { control, receiver } + } + + fn assert_subscribed(&self, expected_params: &SubscriptionParams, expected_id: u64) { + if let NotificationEntry::Subscribed(params, id) = self.receiver.recv().unwrap() { + assert_eq!(¶ms, expected_params); + assert_eq!(id, SubscriptionId::from(expected_id)); + } else { + panic!("unexpected notification"); + } + self.assert_silence(); + } + + fn assert_unsubscribed(&self, expected_params: &SubscriptionParams, expected_id: u64) { + if let NotificationEntry::Unsubscribed(params, id) = self.receiver.recv().unwrap() { + assert_eq!(¶ms, expected_params); + assert_eq!(id, SubscriptionId::from(expected_id)); + } else { + panic!("unexpected notification"); + } + self.assert_silence(); + } + + fn assert_silence(&self) { + assert!(self.receiver.try_recv().is_err()); + } + } + + #[test] + fn notify_subscribe() { + let control = ControlWrapper::new(); + let token1 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 0); + drop(token1); + control.assert_unsubscribed(&SubscriptionParams::Slot, 0); + } + + #[test] + fn notify_subscribe_multiple() { + let control = ControlWrapper::new(); + let token1 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 0); + let token2 = token1.clone(); + drop(token1); + let token3 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + drop(token3); + control.assert_silence(); + drop(token2); + control.assert_unsubscribed(&SubscriptionParams::Slot, 0); + } + + #[test] + fn notify_subscribe_two_subscriptions() { + let control = ControlWrapper::new(); + let token_slot1 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 0); + + let signature_params = SubscriptionParams::Signature(SignatureSubscriptionParams { + signature: Signature::default(), + commitment: CommitmentConfig::processed(), + enable_received_notification: false, + }); + let token_signature1 = control.control.subscribe(signature_params.clone()).unwrap(); + control.assert_subscribed(&signature_params, 1); + + let token_slot2 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + let token_signature2 = control.control.subscribe(signature_params.clone()).unwrap(); + drop(token_slot1); + control.assert_silence(); + drop(token_slot2); + control.assert_unsubscribed(&SubscriptionParams::Slot, 0); + drop(token_signature2); + control.assert_silence(); + drop(token_signature1); + control.assert_unsubscribed(&signature_params, 1); + + let token_slot3 = control.control.subscribe(SubscriptionParams::Slot).unwrap(); + control.assert_subscribed(&SubscriptionParams::Slot, 2); + drop(token_slot3); + control.assert_unsubscribed(&SubscriptionParams::Slot, 2); + } + + #[test] + fn subscription_info() { + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); + let bank = Bank::new(&genesis_config); + let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); + let mut tracker = SubscriptionsTracker::new(bank_forks); + + tracker.subscribe(SubscriptionParams::Slot, 0.into(), || 0); + let info = tracker + .node_progress_watchers + .get(&SubscriptionParams::Slot) + .unwrap(); + assert_eq!(info.commitment, None); + assert_eq!(info.params, SubscriptionParams::Slot); + assert_eq!(info.method, SubscriptionParams::Slot.method()); + assert_eq!(info.id, SubscriptionId::from(0)); + assert_eq!(*info.last_notified_slot.read().unwrap(), 0); + + let account_params = SubscriptionParams::Account(AccountSubscriptionParams { + pubkey: Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), + commitment: CommitmentConfig::finalized(), + encoding: UiAccountEncoding::Base64Zstd, + data_slice: None, + }); + tracker.subscribe(account_params.clone(), 1.into(), || 42); + + let info = tracker + .commitment_watchers + .get(&SubscriptionId::from(1)) + .unwrap(); + assert_eq!(info.commitment, Some(CommitmentConfig::finalized())); + assert_eq!(info.params, account_params); + assert_eq!(info.method, account_params.method()); + assert_eq!(info.id, SubscriptionId::from(1)); + assert_eq!(*info.last_notified_slot.read().unwrap(), 42); + } + + #[test] + fn subscription_indexes() { + fn counts(tracker: &SubscriptionsTracker) -> (usize, usize, usize, usize) { + ( + tracker.by_signature.len(), + tracker.commitment_watchers.len(), + tracker.gossip_watchers.len(), + tracker.node_progress_watchers.len(), + ) + } + + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); + let bank = Bank::new(&genesis_config); + let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); + let mut tracker = SubscriptionsTracker::new(bank_forks); + + tracker.subscribe(SubscriptionParams::Slot, 0.into(), || 0); + assert_eq!(counts(&tracker), (0, 0, 0, 1)); + tracker.unsubscribe(SubscriptionParams::Slot, 0.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + + let account_params = SubscriptionParams::Account(AccountSubscriptionParams { + pubkey: Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), + commitment: CommitmentConfig::finalized(), + encoding: UiAccountEncoding::Base64Zstd, + data_slice: None, + }); + tracker.subscribe(account_params.clone(), 1.into(), || 0); + assert_eq!(counts(&tracker), (0, 1, 0, 0)); + tracker.unsubscribe(account_params, 1.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + + let account_params2 = SubscriptionParams::Account(AccountSubscriptionParams { + pubkey: Pubkey::from_str("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA").unwrap(), + commitment: CommitmentConfig::confirmed(), + encoding: UiAccountEncoding::Base64Zstd, + data_slice: None, + }); + tracker.subscribe(account_params2.clone(), 2.into(), || 0); + assert_eq!(counts(&tracker), (0, 0, 1, 0)); + tracker.unsubscribe(account_params2, 2.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + + let signature_params = SubscriptionParams::Signature(SignatureSubscriptionParams { + signature: Signature::default(), + commitment: CommitmentConfig::processed(), + enable_received_notification: false, + }); + tracker.subscribe(signature_params.clone(), 3.into(), || 0); + assert_eq!(counts(&tracker), (1, 1, 0, 0)); + tracker.unsubscribe(signature_params, 3.into()); + assert_eq!(counts(&tracker), (0, 0, 0, 0)); + } +} diff --git a/core/src/rpc_subscriptions.rs b/core/src/rpc_subscriptions.rs index b9634476a2..ef92c1cb17 100644 --- a/core/src/rpc_subscriptions.rs +++ b/core/src/rpc_subscriptions.rs @@ -3,16 +3,17 @@ use crate::{ optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, rpc::{get_parsed_token_account, get_parsed_token_accounts}, + rpc_pubsub_service::PubSubConfig, + rpc_subscription_tracker::{ + AccountSubscriptionParams, LogsSubscriptionKind, LogsSubscriptionParams, + ProgramSubscriptionParams, SignatureSubscriptionParams, SubscriptionControl, + SubscriptionId, SubscriptionInfo, SubscriptionParams, SubscriptionsTracker, + }, }; -use core::hash::Hash; -use jsonrpc_pubsub::{ - typed::{Sink, Subscriber}, - SubscriptionId, -}; +use crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender}; use serde::Serialize; use solana_account_decoder::{parse_token::spl_token_id_v2_0, UiAccount, UiAccountEncoding}; use solana_client::{ - rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig}, rpc_filter::RpcFilterType, rpc_response::{ ProcessedSignatureResult, ReceivedSignatureResult, Response, RpcKeyedAccount, @@ -21,16 +22,13 @@ use solana_client::{ }; use solana_measure::measure::Measure; use solana_runtime::{ - bank::{ - Bank, TransactionLogCollectorConfig, TransactionLogCollectorFilter, TransactionLogInfo, - }, + bank::{Bank, TransactionLogInfo}, bank_forks::BankForks, commitment::{BlockCommitmentCache, CommitmentSlots}, }; use solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::{Slot, UnixTimestamp}, - commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature, timing::timestamp, @@ -38,39 +36,36 @@ use solana_sdk::{ }; use solana_vote_program::vote_state::Vote; use std::{ - collections::{HashMap, HashSet}, - iter, + collections::{HashMap, VecDeque}, + io::Cursor, + iter, str, sync::{ atomic::{AtomicBool, Ordering}, - mpsc::{Receiver, RecvTimeoutError, SendError, Sender}, + Arc, RwLock, Weak, }, - sync::{Arc, Mutex, RwLock}, thread::{Builder, JoinHandle}, time::Duration, }; +use tokio::sync::broadcast; const RECEIVE_DELAY_MILLIS: u64 = 100; -trait BankGetTransactionLogsAdapter { - fn get_transaction_logs_adapter( - &self, - stuff: &(Option, bool), - ) -> Option>; -} - -impl BankGetTransactionLogsAdapter for Bank { - fn get_transaction_logs_adapter( - &self, - config: &(Option, bool), - ) -> Option> { - let mut logs = self.get_transaction_logs(config.0.as_ref()); - - if config.0.is_none() && !config.1 { - // Filter out votes if the subscriber doesn't want them - logs = logs.map(|logs| logs.into_iter().filter(|log| !log.is_vote).collect()); +fn get_transaction_logs( + bank: &Bank, + params: &LogsSubscriptionParams, +) -> Option> { + let pubkey = match ¶ms.kind { + LogsSubscriptionKind::All | LogsSubscriptionKind::AllWithVotes => None, + LogsSubscriptionKind::Single(pubkey) => Some(pubkey), + }; + let mut logs = bank.get_transaction_logs(pubkey); + if matches!(params.kind, LogsSubscriptionKind::All) { + // Filter out votes if the subscriber doesn't want them + if let Some(logs) = &mut logs { + logs.retain(|log| !log.is_vote); } - logs } + logs } // A more human-friendly version of Vote, with the bank state signature base58 encoded. @@ -81,7 +76,7 @@ pub struct RpcVote { pub timestamp: Option, } -enum NotificationEntry { +pub enum NotificationEntry { Slot(SlotInfo), SlotUpdate(SlotUpdate), Vote(Vote), @@ -89,6 +84,8 @@ enum NotificationEntry { Bank(CommitmentSlots), Gossip(Slot), SignaturesReceived((Slot, Vec)), + Subscribed(SubscriptionParams, SubscriptionId), + Unsubscribed(SubscriptionParams, SubscriptionId), } impl std::fmt::Debug for NotificationEntry { @@ -107,179 +104,174 @@ impl std::fmt::Debug for NotificationEntry { write!(f, "SignaturesReceived({:?})", slot_signatures) } NotificationEntry::Gossip(slot) => write!(f, "Gossip({:?})", slot), + NotificationEntry::Subscribed(params, id) => { + write!(f, "Subscribed({:?}, {:?})", params, id) + } + NotificationEntry::Unsubscribed(params, id) => { + write!(f, "Unsubscribed({:?}, {:?})", params, id) + } } } } -struct SubscriptionData { - sink: Sink, - commitment: CommitmentConfig, - last_notified_slot: RwLock, - config: Option, -} #[derive(Default, Clone)] struct ProgramConfig { filters: Vec, encoding: Option, } -type RpcAccountSubscriptions = RwLock< - HashMap< - Pubkey, - HashMap, UiAccountEncoding>>, - >, ->; -type RpcLogsSubscriptions = RwLock< - HashMap< - (Option, bool), - HashMap, ()>>, - >, ->; -type RpcProgramSubscriptions = RwLock< - HashMap< - Pubkey, - HashMap, ProgramConfig>>, - >, ->; -type RpcSignatureSubscriptions = RwLock< - HashMap< - Signature, - HashMap, bool>>, - >, ->; -type RpcSlotSubscriptions = RwLock>>; -type RpcSlotUpdateSubscriptions = RwLock>>>; -type RpcVoteSubscriptions = RwLock>>; -type RpcRootSubscriptions = RwLock>>; - -fn add_subscription( - subscriptions: &mut HashMap>>, - hashmap_key: K, - commitment: CommitmentConfig, - sub_id: SubscriptionId, - subscriber: Subscriber, - last_notified_slot: Slot, - config: Option, -) where - K: Eq + Hash, - S: Clone, -{ - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let subscription_data = SubscriptionData { - sink, - commitment, - last_notified_slot: RwLock::new(last_notified_slot), - config, - }; - - subscriptions - .entry(hashmap_key) - .or_default() - .insert(sub_id, subscription_data); -} - -fn remove_subscription( - subscriptions: &mut HashMap>>, - sub_id: &SubscriptionId, -) -> bool -where - K: Eq + Hash, - S: Clone, -{ - let mut found = false; - subscriptions.retain(|_, v| { - v.retain(|k, _| { - let retain = k != sub_id; - if !retain { - found = true; - } - retain - }); - !v.is_empty() - }); - found -} #[allow(clippy::type_complexity)] -fn check_commitment_and_notify( - subscriptions: &HashMap, T>>>, - hashmap_key: &K, +fn check_commitment_and_notify( + params: &P, + subscription: &SubscriptionInfo, bank_forks: &Arc>, commitment_slots: &CommitmentSlots, bank_method: B, filter_results: F, - notifier: &RpcNotifier, -) -> HashSet + notifier: &mut RpcNotifier, + is_final: bool, +) -> bool where - K: Eq + Hash + Clone + Copy, S: Clone + Serialize, - B: Fn(&Bank, &K) -> X, - F: Fn(X, &K, Slot, Option, Arc) -> (Box>, Slot), + B: Fn(&Bank, &P) -> X, + F: Fn(X, &P, Slot, Arc) -> (Box>, Slot), X: Clone + Default, - T: Clone, { - let mut notified_set: HashSet = HashSet::new(); - if let Some(hashmap) = subscriptions.get(hashmap_key) { - for ( - sub_id, - SubscriptionData { - sink, - commitment, - last_notified_slot, - config, - }, - ) in hashmap.iter() - { - let slot = if commitment.is_finalized() { - commitment_slots.highest_confirmed_root - } else if commitment.is_confirmed() { - commitment_slots.highest_confirmed_slot - } else { - commitment_slots.slot - }; + let commitment = if let Some(commitment) = subscription.commitment() { + commitment + } else { + error!("missing commitment in check_commitment_and_notify"); + return false; + }; + let slot = if commitment.is_finalized() { + commitment_slots.highest_confirmed_root + } else if commitment.is_confirmed() { + commitment_slots.highest_confirmed_slot + } else { + commitment_slots.slot + }; - if let Some(bank) = bank_forks.read().unwrap().get(slot).cloned() { - let results = bank_method(&bank, hashmap_key); - let mut w_last_notified_slot = last_notified_slot.write().unwrap(); - let (filter_results, result_slot) = filter_results( - results, - hashmap_key, - *w_last_notified_slot, - config.as_ref().cloned(), - bank, - ); - for result in filter_results { - notifier.notify( - Response { - context: RpcResponseContext { slot }, - value: result, - }, - sink, - ); - notified_set.insert(sub_id.clone()); - *w_last_notified_slot = result_slot; - } - } + let mut notified = false; + if let Some(bank) = bank_forks.read().unwrap().get(slot).cloned() { + let results = bank_method(&bank, params); + let mut w_last_notified_slot = subscription.last_notified_slot.write().unwrap(); + let (filter_results, result_slot) = + filter_results(results, params, *w_last_notified_slot, bank); + for result in filter_results { + notifier.notify( + Response { + context: RpcResponseContext { slot }, + value: result, + }, + subscription, + is_final, + ); + *w_last_notified_slot = result_slot; + notified = true; } } - notified_set + notified } -struct RpcNotifier; +#[derive(Debug, Clone)] +pub struct RpcNotification { + pub subscription_id: SubscriptionId, + pub is_final: bool, + pub json: Weak, +} + +struct RecentItems { + queue: VecDeque>, + total_bytes: usize, + max_len: usize, + max_total_bytes: usize, +} + +impl RecentItems { + fn new(max_len: usize, max_total_bytes: usize) -> Self { + Self { + queue: VecDeque::new(), + total_bytes: 0, + max_len, + max_total_bytes, + } + } + + fn push(&mut self, item: Arc) { + self.total_bytes = self + .total_bytes + .checked_add(item.len()) + .expect("total bytes overflow"); + self.queue.push_back(item); + + while self.total_bytes > self.max_total_bytes || self.queue.len() > self.max_len { + let item = self.queue.pop_front().expect("can't be empty"); + self.total_bytes = self + .total_bytes + .checked_sub(item.len()) + .expect("total bytes underflow"); + } + } +} + +struct RpcNotifier { + sender: broadcast::Sender, + buf: Vec, + recent_items: RecentItems, +} + +#[derive(Debug, Serialize)] +struct NotificationParams { + result: T, + subscription: SubscriptionId, +} + +#[derive(Debug, Serialize)] +struct Notification { + jsonrpc: Option, + method: &'static str, + params: NotificationParams, +} impl RpcNotifier { - fn notify(&self, value: T, sink: &Sink) + fn notify(&mut self, value: T, subscription: &SubscriptionInfo, is_final: bool) where T: serde::Serialize, { - let _ = sink.notify(Ok(value)); + self.buf.clear(); + let notification = Notification { + jsonrpc: Some(jsonrpc_core::Version::V2), + method: subscription.method(), + params: NotificationParams { + result: value, + subscription: subscription.id(), + }, + }; + serde_json::to_writer(Cursor::new(&mut self.buf), ¬ification) + .expect("serialization never fails"); + let buf_str = str::from_utf8(&self.buf).expect("json is always utf-8"); + let buf_arc = Arc::new(String::from(buf_str)); + + let notification = RpcNotification { + subscription_id: subscription.id(), + json: Arc::downgrade(&buf_arc), + is_final, + }; + // There is an unlikely case where this can fail: if the last subscription is closed + // just as the notifier generates a notification for it. + let _ = self.sender.send(notification); + + inc_new_counter_info!("rpc-pubsub-messages", 1); + inc_new_counter_info!("rpc-pubsub-bytes", buf_arc.len()); + + self.recent_items.push(buf_arc); } } fn filter_account_result( result: Option<(AccountSharedData, Slot)>, - pubkey: &Pubkey, + params: &AccountSubscriptionParams, last_notified_slot: Slot, - encoding: Option, bank: Arc, ) -> (Box>, Slot) { // If the account is not found, `last_modified_slot` will default to zero and @@ -289,12 +281,20 @@ fn filter_account_result( // If last_modified_slot < last_notified_slot this means that we last notified for a fork // and should notify that the account state has been reverted. let results: Box> = if last_modified_slot != last_notified_slot { - let encoding = encoding.unwrap_or(UiAccountEncoding::Binary); - if account.owner == spl_token_id_v2_0() && encoding == UiAccountEncoding::JsonParsed { - Box::new(iter::once(get_parsed_token_account(bank, pubkey, account))) + if account.owner == spl_token_id_v2_0() && params.encoding == UiAccountEncoding::JsonParsed + { + Box::new(iter::once(get_parsed_token_account( + bank, + ¶ms.pubkey, + account, + ))) } else { Box::new(iter::once(UiAccount::encode( - pubkey, &account, encoding, None, None, + ¶ms.pubkey, + &account, + params.encoding, + None, + None, ))) } } else { @@ -306,9 +306,8 @@ fn filter_account_result( fn filter_signature_result( result: Option>, - _signature: &Signature, + _params: &SignatureSubscriptionParams, last_notified_slot: Slot, - _config: Option, _bank: Arc, ) -> (Box>, Slot) { ( @@ -321,23 +320,22 @@ fn filter_signature_result( fn filter_program_results( accounts: Vec<(Pubkey, AccountSharedData)>, - program_id: &Pubkey, + params: &ProgramSubscriptionParams, last_notified_slot: Slot, - config: Option, bank: Arc, ) -> (Box>, Slot) { - let config = config.unwrap_or_default(); - let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary); - let filters = config.filters; let accounts_is_empty = accounts.is_empty(); + let encoding = params.encoding; + let filters = params.filters.clone(); let keyed_accounts = accounts.into_iter().filter(move |(_, account)| { filters.iter().all(|filter_type| match filter_type { RpcFilterType::DataSize(size) => account.data().len() as u64 == *size, RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data()), }) }); - let accounts: Box> = if program_id == &spl_token_id_v2_0() - && encoding == UiAccountEncoding::JsonParsed + let accounts: Box> = if params.pubkey + == spl_token_id_v2_0() + && params.encoding == UiAccountEncoding::JsonParsed && !accounts_is_empty { Box::new(get_parsed_token_accounts(bank, keyed_accounts)) @@ -354,9 +352,8 @@ fn filter_program_results( fn filter_logs_results( logs: Option>, - _address: &(Option, bool), + _params: &LogsSubscriptionParams, last_notified_slot: Slot, - _config: Option<()>, _bank: Arc, ) -> (Box>, Slot) { match logs { @@ -372,59 +369,54 @@ fn filter_logs_results( } } -fn total_nested_subscriptions( - subscription_map: &RwLock>>, -) -> usize { - subscription_map - .read() - .unwrap() - .iter() - .fold(0, |acc, x| acc + x.1.len()) -} +fn initial_last_notified_slot( + params: &SubscriptionParams, + bank_forks: &RwLock, + block_commitment_cache: &RwLock, + optimistically_confirmed_bank: &RwLock, +) -> Slot { + match params { + SubscriptionParams::Account(params) => { + let slot = if params.commitment.is_finalized() { + block_commitment_cache + .read() + .unwrap() + .highest_confirmed_root() + } else if params.commitment.is_confirmed() { + optimistically_confirmed_bank.read().unwrap().bank.slot() + } else { + block_commitment_cache.read().unwrap().slot() + }; -#[derive(Clone)] -struct Subscriptions { - account_subscriptions: Arc, - program_subscriptions: Arc, - logs_subscriptions: Arc, - signature_subscriptions: Arc, - gossip_account_subscriptions: Arc, - gossip_logs_subscriptions: Arc, - gossip_program_subscriptions: Arc, - gossip_signature_subscriptions: Arc, - slot_subscriptions: Arc, - slots_updates_subscriptions: Arc, - vote_subscriptions: Arc, - root_subscriptions: Arc, -} - -impl Subscriptions { - fn total(&self) -> usize { - let mut total = 0; - total += total_nested_subscriptions(&self.account_subscriptions); - total += total_nested_subscriptions(&self.program_subscriptions); - total += total_nested_subscriptions(&self.logs_subscriptions); - total += total_nested_subscriptions(&self.signature_subscriptions); - total += total_nested_subscriptions(&self.gossip_account_subscriptions); - total += total_nested_subscriptions(&self.gossip_logs_subscriptions); - total += total_nested_subscriptions(&self.gossip_program_subscriptions); - total += total_nested_subscriptions(&self.gossip_signature_subscriptions); - total += self.slot_subscriptions.read().unwrap().len(); - total += self.vote_subscriptions.read().unwrap().len(); - total += self.root_subscriptions.read().unwrap().len(); - total + if let Some((_account, slot)) = bank_forks + .read() + .unwrap() + .get(slot) + .and_then(|bank| bank.get_account_modified_slot(¶ms.pubkey)) + { + slot + } else { + 0 + } + } + // last_notified_slot is not utilized for these subscriptions + SubscriptionParams::Logs(_) + | SubscriptionParams::Program(_) + | SubscriptionParams::Signature(_) + | SubscriptionParams::Slot + | SubscriptionParams::SlotsUpdates + | SubscriptionParams::Root + | SubscriptionParams::Vote => 0, } } pub struct RpcSubscriptions { - subscriptions: Subscriptions, - notification_sender: Arc>>, + notification_sender: Sender, + t_cleanup: Option>, - bank_forks: Arc>, - block_commitment_cache: Arc>, - optimistically_confirmed_bank: Arc>, + exit: Arc, - enable_vote_subscription: bool, + control: SubscriptionControl, } impl Drop for RpcSubscriptions { @@ -442,61 +434,52 @@ impl RpcSubscriptions { block_commitment_cache: Arc>, optimistically_confirmed_bank: Arc>, ) -> Self { - Self::new_with_vote_subscription( + Self::new_with_config( exit, bank_forks, block_commitment_cache, optimistically_confirmed_bank, - false, + &PubSubConfig::default(), ) } - pub fn new_with_vote_subscription( + pub fn new_for_tests( exit: &Arc, bank_forks: Arc>, block_commitment_cache: Arc>, optimistically_confirmed_bank: Arc>, - enable_vote_subscription: bool, ) -> Self { - let (notification_sender, notification_receiver): ( - Sender, - Receiver, - ) = std::sync::mpsc::channel(); + Self::new_with_config( + exit, + bank_forks, + block_commitment_cache, + optimistically_confirmed_bank, + &PubSubConfig::default_for_tests(), + ) + } - let account_subscriptions = Arc::new(RpcAccountSubscriptions::default()); - let logs_subscriptions = Arc::new(RpcLogsSubscriptions::default()); - let program_subscriptions = Arc::new(RpcProgramSubscriptions::default()); - let signature_subscriptions = Arc::new(RpcSignatureSubscriptions::default()); - let gossip_account_subscriptions = Arc::new(RpcAccountSubscriptions::default()); - let gossip_logs_subscriptions = Arc::new(RpcLogsSubscriptions::default()); - let gossip_program_subscriptions = Arc::new(RpcProgramSubscriptions::default()); - let gossip_signature_subscriptions = Arc::new(RpcSignatureSubscriptions::default()); - let slot_subscriptions = Arc::new(RpcSlotSubscriptions::default()); - let slots_updates_subscriptions = Arc::new(RpcSlotUpdateSubscriptions::default()); - let vote_subscriptions = Arc::new(RpcVoteSubscriptions::default()); - let root_subscriptions = Arc::new(RpcRootSubscriptions::default()); - let notification_sender = Arc::new(Mutex::new(notification_sender)); + pub fn new_with_config( + exit: &Arc, + bank_forks: Arc>, + block_commitment_cache: Arc>, + optimistically_confirmed_bank: Arc>, + config: &PubSubConfig, + ) -> Self { + let (notification_sender, notification_receiver) = crossbeam_channel::unbounded(); - let _bank_forks = bank_forks.clone(); - let _block_commitment_cache = block_commitment_cache.clone(); let exit_clone = exit.clone(); - let subscriptions = Subscriptions { - account_subscriptions, - program_subscriptions, - logs_subscriptions, - signature_subscriptions, - gossip_account_subscriptions, - gossip_logs_subscriptions, - gossip_program_subscriptions, - gossip_signature_subscriptions, - slot_subscriptions, - slots_updates_subscriptions, - vote_subscriptions, - root_subscriptions, - }; - let _subscriptions = subscriptions.clone(); + let subscriptions = SubscriptionsTracker::new(bank_forks.clone()); - let notifier = RpcNotifier {}; + let (broadcast_sender, _) = broadcast::channel(config.queue_capacity_items); + + let notifier = RpcNotifier { + sender: broadcast_sender.clone(), + buf: Vec::new(), + recent_items: RecentItems::new( + config.queue_capacity_items, + config.queue_capacity_bytes, + ), + }; let t_cleanup = Builder::new() .name("solana-rpc-notifications".to_string()) .spawn(move || { @@ -504,21 +487,26 @@ impl RpcSubscriptions { exit_clone, notifier, notification_receiver, - _subscriptions, - _bank_forks, + subscriptions, + bank_forks, + block_commitment_cache, + optimistically_confirmed_bank, ); }) .unwrap(); + let control = SubscriptionControl::new( + PubSubConfig::default().max_active_subscriptions, + notification_sender.clone(), + broadcast_sender, + ); + Self { - subscriptions, notification_sender, t_cleanup: Some(t_cleanup), - bank_forks, - block_commitment_cache, - optimistically_confirmed_bank, + exit: exit.clone(), - enable_vote_subscription, + control, } } @@ -526,373 +514,16 @@ impl RpcSubscriptions { pub fn default_with_bank_forks(bank_forks: Arc>) -> Self { let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - Self::new_with_vote_subscription( + Self::new( &Arc::new(AtomicBool::new(false)), bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), optimistically_confirmed_bank, - true, ) } - fn check_account( - pubkey: &Pubkey, - bank_forks: &Arc>, - account_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let subscriptions = account_subscriptions.read().unwrap(); - check_commitment_and_notify( - &subscriptions, - pubkey, - bank_forks, - commitment_slots, - Bank::get_account_modified_slot, - filter_account_result, - notifier, - ) - } - - fn check_logs( - address_with_enable_votes_flag: &(Option, bool), - bank_forks: &Arc>, - logs_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let subscriptions = logs_subscriptions.read().unwrap(); - check_commitment_and_notify( - &subscriptions, - address_with_enable_votes_flag, - bank_forks, - commitment_slots, - Bank::get_transaction_logs_adapter, - filter_logs_results, - notifier, - ) - } - - fn check_program( - program_id: &Pubkey, - bank_forks: &Arc>, - program_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let subscriptions = program_subscriptions.read().unwrap(); - check_commitment_and_notify( - &subscriptions, - program_id, - bank_forks, - commitment_slots, - Bank::get_program_accounts_modified_since_parent, - filter_program_results, - notifier, - ) - } - - fn check_signature( - signature: &Signature, - bank_forks: &Arc>, - signature_subscriptions: Arc, - notifier: &RpcNotifier, - commitment_slots: &CommitmentSlots, - ) -> HashSet { - let mut subscriptions = signature_subscriptions.write().unwrap(); - let notified_ids = check_commitment_and_notify( - &subscriptions, - signature, - bank_forks, - commitment_slots, - Bank::get_signature_status_processed_since_parent, - filter_signature_result, - notifier, - ); - if let Some(subscription_ids) = subscriptions.get_mut(signature) { - subscription_ids.retain(|k, _| !notified_ids.contains(k)); - if subscription_ids.is_empty() { - subscriptions.remove(&signature); - } - } - notified_ids - } - - pub fn total(&self) -> usize { - self.subscriptions.total() - } - - pub fn add_account_subscription( - &self, - pubkey: Pubkey, - config: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let config = config.unwrap_or_default(); - let commitment = config.commitment.unwrap_or_default(); - - let slot = if commitment.is_finalized() { - self.block_commitment_cache - .read() - .unwrap() - .highest_confirmed_root() - } else if commitment.is_confirmed() { - self.optimistically_confirmed_bank - .read() - .unwrap() - .bank - .slot() - } else { - self.block_commitment_cache.read().unwrap().slot() - }; - - let last_notified_slot = if let Some((_account, slot)) = self - .bank_forks - .read() - .unwrap() - .get(slot) - .and_then(|bank| bank.get_account_modified_slot(&pubkey)) - { - slot - } else { - 0 - }; - - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_account_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.account_subscriptions.write().unwrap() - }; - - add_subscription( - &mut subscriptions, - pubkey, - commitment, - sub_id, - subscriber, - last_notified_slot, - config.encoding, - ); - } - - pub fn remove_account_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.account_subscriptions.write().unwrap(); - if remove_subscription(&mut subscriptions, id) { - true - } else { - let mut subscriptions = self - .subscriptions - .gossip_account_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - } - } - - pub fn add_program_subscription( - &self, - program_id: Pubkey, - config: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let config = config.unwrap_or_default(); - let commitment = config.account_config.commitment.unwrap_or_default(); - - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_program_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.program_subscriptions.write().unwrap() - }; - - add_subscription( - &mut subscriptions, - program_id, - commitment, - sub_id, - subscriber, - 0, // last_notified_slot is not utilized for program subscriptions - Some(ProgramConfig { - filters: config.filters.unwrap_or_default(), - encoding: config.account_config.encoding, - }), - ); - } - - pub fn remove_program_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.program_subscriptions.write().unwrap(); - if remove_subscription(&mut subscriptions, id) { - true - } else { - let mut subscriptions = self - .subscriptions - .gossip_program_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - } - } - - pub fn add_logs_subscription( - &self, - address: Option, - include_votes: bool, - commitment: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let commitment = commitment.unwrap_or_default(); - - { - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_logs_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.logs_subscriptions.write().unwrap() - }; - add_subscription( - &mut subscriptions, - (address, include_votes), - commitment, - sub_id, - subscriber, - 0, // last_notified_slot is not utilized for logs subscriptions - None, - ); - } - self.update_bank_transaction_log_keys(); - } - - pub fn remove_logs_subscription(&self, id: &SubscriptionId) -> bool { - let mut removed = { - let mut subscriptions = self.subscriptions.logs_subscriptions.write().unwrap(); - remove_subscription(&mut subscriptions, id) - }; - - if !removed { - removed = { - let mut subscriptions = self - .subscriptions - .gossip_logs_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - }; - } - - if removed { - self.update_bank_transaction_log_keys(); - } - removed - } - - fn update_bank_transaction_log_keys(&self) { - // Grab a write lock for both `logs_subscriptions` and `gossip_logs_subscriptions`, to - // ensure `Bank::transaction_log_collector_config` is updated atomically. - let logs_subscriptions = self.subscriptions.logs_subscriptions.write().unwrap(); - let gossip_logs_subscriptions = self - .subscriptions - .gossip_logs_subscriptions - .write() - .unwrap(); - - let mut config = TransactionLogCollectorConfig::default(); - - let mut all = false; - let mut all_with_votes = false; - let mut mentioned_address = false; - for (address, with_votes) in logs_subscriptions - .keys() - .chain(gossip_logs_subscriptions.keys()) - { - match address { - None => { - if *with_votes { - all_with_votes = true; - } else { - all = true; - } - } - Some(address) => { - config.mentioned_addresses.insert(*address); - mentioned_address = true; - } - } - } - config.filter = if all_with_votes { - TransactionLogCollectorFilter::AllWithVotes - } else if all { - TransactionLogCollectorFilter::All - } else if mentioned_address { - TransactionLogCollectorFilter::OnlyMentionedAddresses - } else { - TransactionLogCollectorFilter::None - }; - - *self - .bank_forks - .read() - .unwrap() - .root_bank() - .transaction_log_collector_config - .write() - .unwrap() = config; - } - - pub fn add_signature_subscription( - &self, - signature: Signature, - signature_subscribe_config: Option, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let (commitment, enable_received_notification) = signature_subscribe_config - .map(|config| (config.commitment, config.enable_received_notification)) - .unwrap_or_default(); - - let commitment = commitment.unwrap_or_default(); - - let mut subscriptions = if commitment.is_confirmed() { - self.subscriptions - .gossip_signature_subscriptions - .write() - .unwrap() - } else { - self.subscriptions.signature_subscriptions.write().unwrap() - }; - - add_subscription( - &mut subscriptions, - signature, - commitment, - sub_id, - subscriber, - 0, // last_notified_slot is not utilized for signature subscriptions - enable_received_notification, - ); - } - - pub fn remove_signature_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.signature_subscriptions.write().unwrap(); - if remove_subscription(&mut subscriptions, id) { - true - } else { - let mut subscriptions = self - .subscriptions - .gossip_signature_subscriptions - .write() - .unwrap(); - remove_subscription(&mut subscriptions, id) - } + pub fn control(&self) -> &SubscriptionControl { + &self.control } /// Notify subscribers of changes to any accounts or new signatures since @@ -911,40 +542,6 @@ impl RpcSubscriptions { self.enqueue_notification(NotificationEntry::SlotUpdate(slot_update)); } - pub fn add_slot_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber) { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self.subscriptions.slot_subscriptions.write().unwrap(); - subscriptions.insert(sub_id, sink); - } - - pub fn remove_slot_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.slot_subscriptions.write().unwrap(); - subscriptions.remove(id).is_some() - } - - pub fn add_slots_updates_subscription( - &self, - sub_id: SubscriptionId, - subscriber: Subscriber>, - ) { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self - .subscriptions - .slots_updates_subscriptions - .write() - .unwrap(); - subscriptions.insert(sub_id, sink); - } - - pub fn remove_slots_updates_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self - .subscriptions - .slots_updates_subscriptions - .write() - .unwrap(); - subscriptions.remove(id).is_some() - } - pub fn notify_slot(&self, slot: Slot, parent: Slot, root: Slot) { self.enqueue_notification(NotificationEntry::Slot(SlotInfo { slot, parent, root })); self.enqueue_notification(NotificationEntry::SlotUpdate(SlotUpdate::CreatedBank { @@ -958,38 +555,10 @@ impl RpcSubscriptions { self.enqueue_notification(NotificationEntry::SignaturesReceived(slot_signatures)); } - pub fn add_vote_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber) { - if self.enable_vote_subscription { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self.subscriptions.vote_subscriptions.write().unwrap(); - subscriptions.insert(sub_id, sink); - } else { - let _ = subscriber.reject(jsonrpc_core::Error::new( - jsonrpc_core::ErrorCode::MethodNotFound, - )); - } - } - - pub fn remove_vote_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.vote_subscriptions.write().unwrap(); - subscriptions.remove(id).is_some() - } - pub fn notify_vote(&self, vote: &Vote) { self.enqueue_notification(NotificationEntry::Vote(vote.clone())); } - pub fn add_root_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber) { - let sink = subscriber.assign_id(sub_id.clone()).unwrap(); - let mut subscriptions = self.subscriptions.root_subscriptions.write().unwrap(); - subscriptions.insert(sub_id, sink); - } - - pub fn remove_root_subscription(&self, id: &SubscriptionId) -> bool { - let mut subscriptions = self.subscriptions.root_subscriptions.write().unwrap(); - subscriptions.remove(id).is_some() - } - pub fn notify_roots(&self, mut rooted_slots: Vec) { rooted_slots.sort_unstable(); rooted_slots.into_iter().for_each(|root| { @@ -1002,12 +571,7 @@ impl RpcSubscriptions { } fn enqueue_notification(&self, notification_entry: NotificationEntry) { - match self - .notification_sender - .lock() - .unwrap() - .send(notification_entry) - { + match self.notification_sender.send(notification_entry) { Ok(()) => (), Err(SendError(notification)) => { warn!( @@ -1020,112 +584,133 @@ impl RpcSubscriptions { fn process_notifications( exit: Arc, - notifier: RpcNotifier, + mut notifier: RpcNotifier, notification_receiver: Receiver, - subscriptions: Subscriptions, + mut subscriptions: SubscriptionsTracker, bank_forks: Arc>, + block_commitment_cache: Arc>, + optimistically_confirmed_bank: Arc>, ) { loop { if exit.load(Ordering::Relaxed) { break; } match notification_receiver.recv_timeout(Duration::from_millis(RECEIVE_DELAY_MILLIS)) { - Ok(notification_entry) => match notification_entry { - NotificationEntry::Slot(slot_info) => { - let subscriptions = subscriptions.slot_subscriptions.read().unwrap(); - let num_subscriptions = subscriptions.len(); - if num_subscriptions > 0 { - debug!( - "slot notify: {:?}, num_subscriptions: {:?}", - slot_info, num_subscriptions - ); + Ok(notification_entry) => { + match notification_entry { + NotificationEntry::Subscribed(params, id) => { + subscriptions.subscribe(params.clone(), id, || { + initial_last_notified_slot( + ¶ms, + &bank_forks, + &block_commitment_cache, + &optimistically_confirmed_bank, + ) + }); } - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-slot", 1); - notifier.notify(slot_info, sink); + NotificationEntry::Unsubscribed(params, id) => { + subscriptions.unsubscribe(params, id); + } + NotificationEntry::Slot(slot_info) => { + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::Slot) + { + debug!("slot notify: {:?}", slot_info); + inc_new_counter_info!("rpc-subscription-notify-slot", 1); + notifier.notify(&slot_info, sub, false); + } + } + NotificationEntry::SlotUpdate(slot_update) => { + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::SlotsUpdates) + { + inc_new_counter_info!("rpc-subscription-notify-slots-updates", 1); + notifier.notify(&slot_update, sub, false); + } + } + // These notifications are only triggered by votes observed on gossip, + // unlike `NotificationEntry::Gossip`, which also accounts for slots seen + // in VoteState's from bank states built in ReplayStage. + NotificationEntry::Vote(ref vote_info) => { + let rpc_vote = RpcVote { + // TODO: Remove clones + slots: vote_info.slots.clone(), + hash: bs58::encode(vote_info.hash).into_string(), + timestamp: vote_info.timestamp, + }; + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::Vote) + { + debug!("vote notify: {:?}", vote_info); + inc_new_counter_info!("rpc-subscription-notify-vote", 1); + notifier.notify(&rpc_vote, sub, false); + } + } + NotificationEntry::Root(root) => { + if let Some(sub) = subscriptions + .node_progress_watchers() + .get(&SubscriptionParams::Root) + { + debug!("root notify: {:?}", root); + inc_new_counter_info!("rpc-subscription-notify-root", 1); + notifier.notify(&root, sub, false); + } + } + NotificationEntry::Bank(commitment_slots) => { + RpcSubscriptions::notify_accounts_logs_programs_signatures( + subscriptions.commitment_watchers(), + &bank_forks, + &commitment_slots, + &mut notifier, + "bank", + ) + } + NotificationEntry::Gossip(slot) => { + let commitment_slots = CommitmentSlots { + highest_confirmed_slot: slot, + ..CommitmentSlots::default() + }; + + RpcSubscriptions::notify_accounts_logs_programs_signatures( + subscriptions.gossip_watchers(), + &bank_forks, + &commitment_slots, + &mut notifier, + "gossip", + ) + } + NotificationEntry::SignaturesReceived((slot, slot_signatures)) => { + for slot_signature in &slot_signatures { + if let Some(subs) = subscriptions.by_signature().get(slot_signature) + { + for subscription in subs.values() { + if let SubscriptionParams::Signature(params) = + subscription.params() + { + if params.enable_received_notification { + notifier.notify( + Response { + context: RpcResponseContext { slot }, + value: RpcSignatureResult::ReceivedSignature( + ReceivedSignatureResult::ReceivedSignature, + ), + }, + subscription, + false, + ); + } + } else { + error!("invalid params type in visit_by_signature"); + } + } + } + } } } - NotificationEntry::SlotUpdate(slot_update) => { - let subscriptions = - subscriptions.slots_updates_subscriptions.read().unwrap(); - let slot_update = Arc::new(slot_update); - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-slots-updates", 1); - notifier.notify(slot_update.clone(), sink); - } - } - // These notifications are only triggered by votes observed on gossip, - // unlike `NotificationEntry::Gossip`, which also accounts for slots seen - // in VoteState's from bank states built in ReplayStage. - NotificationEntry::Vote(ref vote_info) => { - let subscriptions = subscriptions.vote_subscriptions.read().unwrap(); - let num_subscriptions = subscriptions.len(); - if num_subscriptions > 0 { - debug!( - "vote notify: {:?}, num_subscriptions: {:?}", - vote_info, num_subscriptions - ); - } - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-vote", 1); - notifier.notify( - RpcVote { - // TODO: Remove clones - slots: vote_info.slots.clone(), - hash: bs58::encode(vote_info.hash).into_string(), - timestamp: vote_info.timestamp, - }, - sink, - ); - } - } - NotificationEntry::Root(root) => { - let subscriptions = subscriptions.root_subscriptions.read().unwrap(); - let num_subscriptions = subscriptions.len(); - if num_subscriptions > 0 { - debug!( - "root notify: {:?}, num_subscriptions: {:?}", - root, num_subscriptions - ); - } - for (_, sink) in subscriptions.iter() { - inc_new_counter_info!("rpc-subscription-notify-root", 1); - notifier.notify(root, sink); - } - } - NotificationEntry::Bank(commitment_slots) => { - RpcSubscriptions::notify_accounts_logs_programs_signatures( - &subscriptions.account_subscriptions, - &subscriptions.logs_subscriptions, - &subscriptions.program_subscriptions, - &subscriptions.signature_subscriptions, - &bank_forks, - &commitment_slots, - ¬ifier, - "bank", - ) - } - NotificationEntry::Gossip(slot) => { - Self::process_gossip_notification( - slot, - ¬ifier, - &subscriptions, - &bank_forks, - ); - } - NotificationEntry::SignaturesReceived(slot_signatures) => { - RpcSubscriptions::process_signatures_received( - &slot_signatures, - &subscriptions.gossip_signature_subscriptions, - ¬ifier, - ); - RpcSubscriptions::process_signatures_received( - &slot_signatures, - &subscriptions.signature_subscriptions, - ¬ifier, - ); - } - }, + } Err(RecvTimeoutError::Timeout) => { // not a problem - try reading again } @@ -1137,176 +722,155 @@ impl RpcSubscriptions { } } - fn process_gossip_notification( - slot: Slot, - notifier: &RpcNotifier, - subscriptions: &Subscriptions, - bank_forks: &Arc>, - ) { - let commitment_slots = CommitmentSlots { - highest_confirmed_slot: slot, - ..CommitmentSlots::default() - }; - RpcSubscriptions::notify_accounts_logs_programs_signatures( - &subscriptions.gossip_account_subscriptions, - &subscriptions.gossip_logs_subscriptions, - &subscriptions.gossip_program_subscriptions, - &subscriptions.gossip_signature_subscriptions, - bank_forks, - &commitment_slots, - ¬ifier, - "gossip", - ); - } - fn notify_accounts_logs_programs_signatures( - account_subscriptions: &Arc, - logs_subscriptions: &Arc, - program_subscriptions: &Arc, - signature_subscriptions: &Arc, + subscriptions: &HashMap>, bank_forks: &Arc>, commitment_slots: &CommitmentSlots, - notifier: &RpcNotifier, + notifier: &mut RpcNotifier, source: &'static str, ) { - let mut accounts_time = Measure::start("accounts"); - let pubkeys: Vec<_> = { - let subs = account_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; - let mut num_pubkeys_notified = 0; - for pubkey in &pubkeys { - num_pubkeys_notified += Self::check_account( - pubkey, - bank_forks, - account_subscriptions.clone(), - ¬ifier, - &commitment_slots, - ) - .len(); - } - accounts_time.stop(); + let mut total_time = Measure::start("notify_accounts_logs_programs_signatures"); + let mut num_accounts_found = 0; + let mut num_accounts_notified = 0; - let mut logs_time = Measure::start("logs"); - let logs: Vec<_> = { - let subs = logs_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; + let mut num_logs_found = 0; let mut num_logs_notified = 0; - for address in &logs { - num_logs_notified += Self::check_logs( - address, - bank_forks, - logs_subscriptions.clone(), - ¬ifier, - &commitment_slots, - ) - .len(); - } - logs_time.stop(); - let mut programs_time = Measure::start("programs"); - let programs: Vec<_> = { - let subs = program_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; - let mut num_programs_notified = 0; - for program_id in &programs { - num_programs_notified += Self::check_program( - program_id, - bank_forks, - program_subscriptions.clone(), - ¬ifier, - &commitment_slots, - ) - .len(); - } - programs_time.stop(); - - let mut signatures_time = Measure::start("signatures"); - let signatures: Vec<_> = { - let subs = signature_subscriptions.read().unwrap(); - subs.keys().cloned().collect() - }; + let mut num_signatures_found = 0; let mut num_signatures_notified = 0; - for signature in &signatures { - num_signatures_notified += Self::check_signature( - signature, - bank_forks, - signature_subscriptions.clone(), - ¬ifier, - &commitment_slots, - ) - .len(); + + let mut num_programs_found = 0; + let mut num_programs_notified = 0; + + for subscription in subscriptions.values() { + match subscription.params() { + SubscriptionParams::Account(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + |bank, params| bank.get_account_modified_slot(¶ms.pubkey), + filter_account_result, + notifier, + false, + ); + + num_accounts_found += 1; + + if notified { + num_accounts_notified += 1; + } + } + SubscriptionParams::Logs(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + get_transaction_logs, + filter_logs_results, + notifier, + false, + ); + num_logs_found += 1; + + if notified { + num_logs_notified += 1; + } + } + SubscriptionParams::Program(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + |bank, params| { + bank.get_program_accounts_modified_since_parent(¶ms.pubkey) + }, + filter_program_results, + notifier, + false, + ); + num_programs_found += 1; + + if notified { + num_programs_notified += 1; + } + } + SubscriptionParams::Signature(params) => { + let notified = check_commitment_and_notify( + params, + subscription, + bank_forks, + commitment_slots, + |bank, params| { + bank.get_signature_status_processed_since_parent(¶ms.signature) + }, + filter_signature_result, + notifier, + true, // Unsubscribe. + ); + num_signatures_found += 1; + + if notified { + num_signatures_notified += 1; + } + } + _ => error!("wrong subscription type in alps map"), + } } - signatures_time.stop(); - let total_notified = num_pubkeys_notified + num_programs_notified + num_signatures_notified; - let total_ms = accounts_time.as_ms() + programs_time.as_ms() + signatures_time.as_ms(); + + total_time.stop(); + + let total_notified = num_accounts_notified + + num_logs_notified + + num_programs_notified + + num_signatures_notified; + let total_ms = total_time.as_ms(); if total_notified > 0 || total_ms > 10 { debug!( - "notified({}): accounts: {} / {} ({}) programs: {} / {} ({}) signatures: {} / {} ({})", + "notified({}): accounts: {} / {} logs: {} / {} programs: {} / {} signatures: {} / {}", source, - pubkeys.len(), - num_pubkeys_notified, - accounts_time, - programs.len(), + num_accounts_found, + num_accounts_notified, + num_logs_found, + num_logs_notified, + num_programs_found, num_programs_notified, - programs_time, - signatures.len(), + num_signatures_found, num_signatures_notified, - signatures_time, ); inc_new_counter_info!("rpc-subscription-notify-bank-or-gossip", total_notified); datapoint_info!( "rpc_subscriptions", ("source", source.to_string(), String), - ("num_account_subscriptions", pubkeys.len(), i64), - ("num_account_pubkeys_notified", num_pubkeys_notified, i64), - ("accounts_time", accounts_time.as_us() as i64, i64), - ("num_logs_subscriptions", logs.len(), i64), + ("num_account_subscriptions", num_accounts_found, i64), + ("num_account_pubkeys_notified", num_accounts_notified, i64), + ("num_logs_subscriptions", num_logs_found, i64), ("num_logs_notified", num_logs_notified, i64), - ("logs_time", logs_time.as_us() as i64, i64), - ("num_program_subscriptions", programs.len(), i64), + ("num_program_subscriptions", num_programs_found, i64), ("num_programs_notified", num_programs_notified, i64), - ("programs_time", programs_time.as_us() as i64, i64), - ("num_signature_subscriptions", signatures.len(), i64), + ("num_signature_subscriptions", num_signatures_found, i64), ("num_signatures_notified", num_signatures_notified, i64), - ("signatures_time", signatures_time.as_us() as i64, i64) + ("notifications_time", total_time.as_us() as i64, i64), + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_accounts_notified", + num_accounts_notified + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_logs_notified", + num_logs_notified + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_programs_notified", + num_programs_notified + ); + inc_new_counter_info!( + "rpc-subscription-counter-num_signatures_notified", + num_signatures_notified ); - } - } - - fn process_signatures_received( - (received_slot, signatures): &(Slot, Vec), - signature_subscriptions: &Arc, - notifier: &RpcNotifier, - ) { - for signature in signatures { - if let Some(hashmap) = signature_subscriptions.read().unwrap().get(signature) { - for ( - _, - SubscriptionData { - sink, - config: is_received_notification_enabled, - .. - }, - ) in hashmap.iter() - { - if is_received_notification_enabled.unwrap_or_default() { - notifier.notify( - Response { - context: RpcResponseContext { - slot: *received_slot, - }, - value: RpcSignatureResult::ReceivedSignature( - ReceivedSignatureResult::ReceivedSignature, - ), - }, - &sink, - ); - } - } - } } } @@ -1322,62 +886,59 @@ impl RpcSubscriptions { Ok(()) } } + + #[cfg(test)] + fn total(&self) -> usize { + self.control.total() + } } #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::optimistically_confirmed_bank_tracker::{ - BankNotification, OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker, + use crate::{ + optimistically_confirmed_bank_tracker::{ + BankNotification, OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker, + }, + rpc_pubsub::RpcSolPubSubInternal, + rpc_pubsub_service, }; - use jsonrpc_core::futures::StreamExt; - use jsonrpc_pubsub::typed::Subscriber; use serial_test::serial; + use solana_client::rpc_config::{ + RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig, + RpcTransactionLogsFilter, + }; use solana_runtime::{ commitment::BlockCommitment, genesis_utils::{create_genesis_config, GenesisConfigInfo}, }; use solana_sdk::{ + commitment_config::CommitmentConfig, message::Message, signature::{Keypair, Signer}, system_instruction, system_program, system_transaction, transaction::Transaction, }; - use std::{ - fmt::Debug, - sync::{atomic::Ordering::Relaxed, mpsc::channel}, - }; - use tokio::{ - runtime::Runtime, - time::{sleep, timeout}, - }; + use std::{collections::HashSet, sync::atomic::Ordering::Relaxed}; - pub(crate) fn robust_poll_or_panic( - receiver: jsonrpc_core::futures::channel::mpsc::UnboundedReceiver, - ) -> ( - T, - jsonrpc_core::futures::channel::mpsc::UnboundedReceiver, - ) { - let (inner_sender, inner_receiver) = channel(); - let rt = Runtime::new().unwrap(); - rt.spawn(async move { - let result = timeout( - Duration::from_millis(RECEIVE_DELAY_MILLIS), - receiver.into_future(), - ) - .await - .unwrap_or_else(|err| panic!("stream error {:?}", err)); - - match result { - (Some(value), receiver) => { - inner_sender.send((value, receiver)).expect("send error") - } - (None, _) => panic!("unexpected end of stream"), - } - - sleep(Duration::from_millis(RECEIVE_DELAY_MILLIS * 2)).await; - }); - inner_receiver.recv().expect("recv error") + fn make_account_result(lamports: u64, subscription: u64, data: &str) -> serde_json::Value { + json!({ + "jsonrpc": "2.0", + "method": "accountNotification", + "params": { + "result": { + "context": { "slot": 1 }, + "value": { + "data": data, + "executable": false, + "lamports": lamports, + "owner": "11111111111111111111111111111111", + "rentEpoch": 0, + }, + }, + "subscription": subscription, + } + }) } #[test] @@ -1396,40 +957,17 @@ pub(crate) mod tests { bank_forks.write().unwrap().insert(bank1); let alice = Keypair::new(); - let (create_sub, _id_receiver, create_recv) = Subscriber::new_test("accountNotification"); - let (close_sub, _id_receiver, close_recv) = Subscriber::new_test("accountNotification"); - - let create_sub_id = SubscriptionId::Number(0); - let close_sub_id = SubscriptionId::Number(1); - let exit = Arc::new(AtomicBool::new(false)); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( 1, 1, ))), OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks), - ); - subscriptions.add_account_subscription( - alice.pubkey(), - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::processed()), - encoding: None, - data_slice: None, - }), - create_sub_id.clone(), - create_sub, - ); + )); - assert!(subscriptions - .subscriptions - .account_subscriptions - .read() - .unwrap() - .contains_key(&alice.pubkey())); - - let tx = system_transaction::create_account( + let tx0 = system_transaction::create_account( &mint_keypair, &alice, blockhash, @@ -1437,92 +975,85 @@ pub(crate) mod tests { 0, &system_program::id(), ); - bank_forks - .write() - .unwrap() - .get(1) - .unwrap() - .process_transaction(&tx) - .unwrap(); - let commitment_slots = CommitmentSlots { - slot: 1, - ..CommitmentSlots::default() - }; - subscriptions.notify_subscribers(commitment_slots); - let (response, _) = robust_poll_or_panic(create_recv); - let expected = json!({ - "jsonrpc": "2.0", - "method": "accountNotification", - "params": { - "result": { - "context": { "slot": 1 }, - "value": { - "data": "", - "executable": false, - "lamports": 1, - "owner": "11111111111111111111111111111111", - "rentEpoch": 0, - }, - }, - "subscription": 0, - } - }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&create_sub_id); + let expected0 = make_account_result(1, 0, ""); - subscriptions.add_account_subscription( - alice.pubkey(), - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::processed()), - encoding: None, - data_slice: None, - }), - close_sub_id.clone(), - close_sub, - ); - - let tx = { + let tx1 = { let instruction = system_instruction::transfer(&alice.pubkey(), &mint_keypair.pubkey(), 1); let message = Message::new(&[instruction], Some(&mint_keypair.pubkey())); Transaction::new(&[&alice, &mint_keypair], message, blockhash) }; + let expected1 = make_account_result(0, 1, ""); - bank_forks - .write() - .unwrap() - .get(1) - .unwrap() - .process_transaction(&tx) - .unwrap(); - subscriptions.notify_subscribers(commitment_slots); - let (response, _) = robust_poll_or_panic(close_recv); - let expected = json!({ - "jsonrpc": "2.0", - "method": "accountNotification", - "params": { - "result": { - "context": { "slot": 1 }, - "value": { - "data": "", - "executable": false, - "lamports": 0, - "owner": "11111111111111111111111111111111", - "rentEpoch": 0, - }, - }, - "subscription": 1, - } - }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&close_sub_id); + let tx2 = system_transaction::create_account( + &mint_keypair, + &alice, + blockhash, + 1, + 1024, + &system_program::id(), + ); + let expected_data = [0; 1024]; + let expected2 = make_account_result(1, 2, &bs58::encode(&expected_data).into_string()); - assert!(!subscriptions - .subscriptions - .account_subscriptions - .read() - .unwrap() - .contains_key(&alice.pubkey())); + let subscribe_cases = vec![ + (alice.pubkey(), tx0, expected0), + (alice.pubkey(), tx1, expected1), + (alice.pubkey(), tx2, expected2), + ]; + + for (pubkey, tx, expected) in subscribe_cases { + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + + let sub_id = rpc + .account_subscribe( + pubkey.to_string(), + Some(RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::processed()), + encoding: None, + data_slice: None, + }), + ) + .unwrap(); + + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Account(AccountSubscriptionParams { + pubkey, + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + })); + + bank_forks + .read() + .unwrap() + .get(1) + .unwrap() + .process_transaction(&tx) + .unwrap(); + let commitment_slots = CommitmentSlots { + slot: 1, + ..CommitmentSlots::default() + }; + subscriptions.notify_subscribers(commitment_slots); + let response = receiver.recv(); + + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc.account_unsubscribe(sub_id).unwrap(); + + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Account(AccountSubscriptionParams { + pubkey, + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + })); + } } #[test] @@ -1553,40 +1084,42 @@ pub(crate) mod tests { .process_transaction(&tx) .unwrap(); - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, - ); - subscriptions.add_program_subscription( - solana_stake_program::id(), - Some(RpcProgramAccountsConfig { - account_config: RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::processed()), - ..RpcAccountInfoConfig::default() - }, - ..RpcProgramAccountsConfig::default() - }), - sub_id.clone(), - subscriber, - ); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc + .program_subscribe( + solana_stake_program::id().to_string(), + Some(RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::processed()), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }), + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .program_subscriptions - .read() - .unwrap() - .contains_key(&solana_stake_program::id())); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: solana_stake_program::id(), + filters: Vec::new(), + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + with_context: false, + })); subscriptions.notify_subscribers(CommitmentSlots::default()); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = json!({ "jsonrpc": "2.0", "method": "programNotification", @@ -1607,15 +1140,22 @@ pub(crate) mod tests { "subscription": 0, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - subscriptions.remove_program_subscription(&sub_id); - assert!(!subscriptions - .subscriptions - .program_subscriptions - .read() - .unwrap() - .contains_key(&solana_stake_program::id())); + rpc.program_unsubscribe(sub_id).unwrap(); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: solana_stake_program::id(), + filters: Vec::new(), + commitment: CommitmentConfig::processed(), + data_slice: None, + encoding: UiAccountEncoding::Binary, + with_context: false, + })); } #[test] @@ -1688,15 +1228,12 @@ pub(crate) mod tests { bank3.process_transaction(&tx).unwrap(); // now add programSubscribe at the "confirmed" commitment level - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -1704,25 +1241,32 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - subscriptions.add_program_subscription( - solana_stake_program::id(), - Some(RpcProgramAccountsConfig { - account_config: RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - ..RpcAccountInfoConfig::default() - }, - ..RpcProgramAccountsConfig::default() - }), - sub_id.clone(), - subscriber, - ); - assert!(subscriptions - .subscriptions - .gossip_program_subscriptions - .read() - .unwrap() - .contains_key(&solana_stake_program::id())); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + + let sub_id = rpc + .program_subscribe( + solana_stake_program::id().to_string(), + Some(RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }), + ) + .unwrap(); + + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: solana_stake_program::id(), + filters: Vec::new(), + encoding: UiAccountEncoding::Binary, + data_slice: None, + commitment: CommitmentConfig::confirmed(), + with_context: false, + })); let mut highest_confirmed_slot: Slot = 0; let mut last_notified_confirmed_slot: Slot = 0; @@ -1762,13 +1306,19 @@ pub(crate) mod tests { }) }; - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(1, 1, &alice.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(2, 2, &bob.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); bank3.freeze(); OptimisticallyConfirmedBankTracker::process_notification( @@ -1781,10 +1331,13 @@ pub(crate) mod tests { &mut highest_confirmed_slot, ); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(3, 3, &joe.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_program_subscription(&sub_id); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc.program_unsubscribe(sub_id).unwrap(); } #[test] @@ -1842,15 +1395,12 @@ pub(crate) mod tests { bank2.process_transaction(&tx).unwrap(); // now add programSubscribe at the "confirmed" commitment level - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -1858,8 +1408,9 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - subscriptions.add_program_subscription( - solana_stake_program::id(), + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + rpc.program_subscribe( + solana_stake_program::id().to_string(), Some(RpcProgramAccountsConfig { account_config: RpcAccountInfoConfig { commitment: Some(CommitmentConfig::confirmed()), @@ -1867,16 +1418,19 @@ pub(crate) mod tests { }, ..RpcProgramAccountsConfig::default() }), - sub_id, - subscriber, - ); + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .gossip_program_subscriptions - .read() - .unwrap() - .contains_key(&solana_stake_program::id())); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: solana_stake_program::id(), + filters: Vec::new(), + encoding: UiAccountEncoding::Binary, + data_slice: None, + commitment: CommitmentConfig::confirmed(), + with_context: false, + })); let mut highest_confirmed_slot: Slot = 0; let mut last_notified_confirmed_slot: Slot = 0; @@ -1893,7 +1447,7 @@ pub(crate) mod tests { ); // The following should panic - let (_response, _transport_receiver) = robust_poll_or_panic(transport_receiver); + let _response = receiver.recv(); } #[test] @@ -1950,15 +1504,12 @@ pub(crate) mod tests { bank2.process_transaction(&tx).unwrap(); // now add programSubscribe at the "confirmed" commitment level - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("programNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -1966,25 +1517,30 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - subscriptions.add_program_subscription( - solana_stake_program::id(), - Some(RpcProgramAccountsConfig { - account_config: RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - ..RpcAccountInfoConfig::default() - }, - ..RpcProgramAccountsConfig::default() - }), - sub_id.clone(), - subscriber, - ); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc + .program_subscribe( + solana_stake_program::id().to_string(), + Some(RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }), + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .gossip_program_subscriptions - .read() - .unwrap() - .contains_key(&solana_stake_program::id())); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Program(ProgramSubscriptionParams { + pubkey: solana_stake_program::id(), + filters: Vec::new(), + encoding: UiAccountEncoding::Binary, + data_slice: None, + commitment: CommitmentConfig::confirmed(), + with_context: false, + })); let mut highest_confirmed_slot: Slot = 0; let mut last_notified_confirmed_slot: Slot = 0; @@ -2052,18 +1608,27 @@ pub(crate) mod tests { &mut highest_confirmed_slot, ); - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(1, 1, &alice.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - let (response, transport_receiver) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(2, 2, &bob.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); let expected = build_expected_resp(3, 3, &joe.pubkey().to_string(), 0); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_program_subscription(&sub_id); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc.program_unsubscribe(sub_id).unwrap(); } #[test] @@ -2127,79 +1692,78 @@ pub(crate) mod tests { let exit = Arc::new(AtomicBool::new(false)); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(block_commitment_cache)), optimistically_confirmed_bank, - ); + )); - let (past_bank_sub1, _id_receiver, past_bank_recv1) = - Subscriber::new_test("signatureNotification"); - let (past_bank_sub2, _id_receiver, past_bank_recv2) = - Subscriber::new_test("signatureNotification"); - let (processed_sub, _id_receiver, processed_recv) = - Subscriber::new_test("signatureNotification"); - let (processed_sub3, _id_receiver, processed_recv3) = - Subscriber::new_test("signatureNotification"); + let (past_bank_rpc1, mut past_bank_receiver1) = + rpc_pubsub_service::test_connection(&subscriptions); + let (past_bank_rpc2, mut past_bank_receiver2) = + rpc_pubsub_service::test_connection(&subscriptions); + let (processed_rpc, mut processed_receiver) = + rpc_pubsub_service::test_connection(&subscriptions); + let (another_rpc, _another_receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let (processed_rpc3, mut processed_receiver3) = + rpc_pubsub_service::test_connection(&subscriptions); + + let past_bank_sub_id1 = past_bank_rpc1 + .signature_subscribe( + past_bank_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); + let past_bank_sub_id2 = past_bank_rpc2 + .signature_subscribe( + past_bank_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::finalized()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); + let processed_sub_id = processed_rpc + .signature_subscribe( + processed_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); + another_rpc + .signature_subscribe( + unprocessed_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(false), + }), + ) + .unwrap(); - subscriptions.add_signature_subscription( - past_bank_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(1), - past_bank_sub1, - ); - subscriptions.add_signature_subscription( - past_bank_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::finalized()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(2), - past_bank_sub2, - ); - subscriptions.add_signature_subscription( - processed_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(3), - processed_sub, - ); - subscriptions.add_signature_subscription( - unprocessed_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(false), - }), - SubscriptionId::Number(4), - Subscriber::new_test("signatureNotification").0, - ); // Add a subscription that gets `received` notifications - subscriptions.add_signature_subscription( - unprocessed_tx.signatures[0], - Some(RpcSignatureSubscribeConfig { - commitment: Some(CommitmentConfig::processed()), - enable_received_notification: Some(true), - }), - SubscriptionId::Number(5), - processed_sub3, - ); + let processed_sub_id3 = processed_rpc3 + .signature_subscribe( + unprocessed_tx.signatures[0].to_string(), + Some(RpcSignatureSubscribeConfig { + commitment: Some(CommitmentConfig::processed()), + enable_received_notification: Some(true), + }), + ) + .unwrap(); + + assert!(subscriptions + .control + .signature_subscribed(&unprocessed_tx.signatures[0])); + assert!(subscriptions + .control + .signature_subscribed(&processed_tx.signatures[0])); - { - let sig_subs = subscriptions - .subscriptions - .signature_subscriptions - .read() - .unwrap(); - assert_eq!(sig_subs.get(&past_bank_tx.signatures[0]).unwrap().len(), 2); - assert!(sig_subs.contains_key(&unprocessed_tx.signatures[0])); - assert!(sig_subs.contains_key(&processed_tx.signatures[0])); - } let mut commitment_slots = CommitmentSlots::default(); let received_slot = 1; commitment_slots.slot = received_slot; @@ -2233,130 +1797,135 @@ pub(crate) mod tests { // Expect to receive a notification from bank 1 because this subscription is // looking for 0 confirmations and so checks the current bank - let expected = expected_notification(Notification { slot: 1, id: 1 }, &expected_res); - let (response, _) = robust_poll_or_panic(past_bank_recv1); + let expected = expected_notification( + Notification { + slot: 1, + id: past_bank_sub_id1.into(), + }, + &expected_res, + ); + let response = past_bank_receiver1.recv(); assert_eq!(expected, response); // Expect to receive a notification from bank 0 because this subscription is // looking for 1 confirmation and so checks the past bank - let expected = expected_notification(Notification { slot: 0, id: 2 }, &expected_res); - let (response, _) = robust_poll_or_panic(past_bank_recv2); + let expected = expected_notification( + Notification { + slot: 0, + id: past_bank_sub_id2.into(), + }, + &expected_res, + ); + let response = past_bank_receiver2.recv(); assert_eq!(expected, response); - let expected = expected_notification(Notification { slot: 1, id: 3 }, &expected_res); - let (response, _) = robust_poll_or_panic(processed_recv); + let expected = expected_notification( + Notification { + slot: 1, + id: processed_sub_id.into(), + }, + &expected_res, + ); + let response = processed_receiver.recv(); assert_eq!(expected, response); // Expect a "received" notification let expected = expected_notification( Notification { slot: received_slot, - id: 5, + id: processed_sub_id3.into(), }, &received_expected_res, ); - let (response, _) = robust_poll_or_panic(processed_recv3); + let response = processed_receiver3.recv(); assert_eq!(expected, response); // Subscription should be automatically removed after notification - let sig_subs = subscriptions - .subscriptions - .signature_subscriptions - .read() - .unwrap(); - assert!(!sig_subs.contains_key(&processed_tx.signatures[0])); - assert!(!sig_subs.contains_key(&past_bank_tx.signatures[0])); + + assert!(!subscriptions + .control + .signature_subscribed(&processed_tx.signatures[0])); + assert!(!subscriptions + .control + .signature_subscribed(&past_bank_tx.signatures[0])); // Unprocessed signature subscription should not be removed - assert_eq!( - sig_subs.get(&unprocessed_tx.signatures[0]).unwrap().len(), - 2 - ); + assert!(subscriptions + .control + .signature_subscribed(&unprocessed_tx.signatures[0])); } #[test] #[serial] fn test_check_slot_subscribe() { - let (subscriber, _id_receiver, transport_receiver) = - Subscriber::new_test("slotNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, - ); - subscriptions.add_slot_subscription(sub_id.clone(), subscriber); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc.slot_subscribe().unwrap(); - assert!(subscriptions - .subscriptions - .slot_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Slot); subscriptions.notify_slot(0, 0, 0); - let (response, _) = robust_poll_or_panic(transport_receiver); + let response = receiver.recv(); + let expected_res = SlotInfo { parent: 0, slot: 0, root: 0, }; - let expected_res_str = - serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap(); + let expected_res_str = serde_json::to_string(&expected_res).unwrap(); + let expected = format!( r#"{{"jsonrpc":"2.0","method":"slotNotification","params":{{"result":{},"subscription":0}}}}"#, expected_res_str ); assert_eq!(expected, response); - subscriptions.remove_slot_subscription(&sub_id); - assert!(!subscriptions - .subscriptions - .slot_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); + rpc.slot_unsubscribe(sub_id).unwrap(); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Slot); } #[test] #[serial] fn test_check_root_subscribe() { - let (subscriber, _id_receiver, mut transport_receiver) = - Subscriber::new_test("rootNotification"); - let sub_id = SubscriptionId::Number(0); let exit = Arc::new(AtomicBool::new(false)); let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let bank = Bank::new(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())), optimistically_confirmed_bank, - ); - subscriptions.add_root_subscription(sub_id.clone(), subscriber); + )); + let (rpc, mut receiver) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id = rpc.root_subscribe().unwrap(); - assert!(subscriptions - .subscriptions - .root_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); + subscriptions + .control + .assert_subscribed(&SubscriptionParams::Root); subscriptions.notify_roots(vec![2, 1, 3]); for expected_root in 1..=3 { - let (response, receiver) = robust_poll_or_panic(transport_receiver); - transport_receiver = receiver; + let response = receiver.recv(); + let expected_res_str = serde_json::to_string(&serde_json::to_value(expected_root).unwrap()).unwrap(); let expected = format!( @@ -2366,69 +1935,10 @@ pub(crate) mod tests { assert_eq!(expected, response); } - subscriptions.remove_root_subscription(&sub_id); - assert!(!subscriptions - .subscriptions - .root_subscriptions - .read() - .unwrap() - .contains_key(&sub_id)); - } - - #[test] - #[serial] - fn test_add_and_remove_subscription() { - let mut subscriptions: HashMap>> = - HashMap::new(); - let commitment = CommitmentConfig::confirmed(); - - let num_keys = 5; - for key in 0..num_keys { - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("notification"); - let sub_id = SubscriptionId::Number(key); - add_subscription( - &mut subscriptions, - key, - commitment, - sub_id, - subscriber, - 0, - None, - ); - } - - // Add another subscription to the "0" key - let (subscriber, _id_receiver, _transport_receiver) = Subscriber::new_test("notification"); - let extra_sub_id = SubscriptionId::Number(num_keys); - add_subscription( - &mut subscriptions, - 0, - commitment, - extra_sub_id.clone(), - subscriber, - 0, - None, - ); - - assert_eq!(subscriptions.len(), num_keys as usize); - assert_eq!(subscriptions.get(&0).unwrap().len(), 2); - assert_eq!(subscriptions.get(&1).unwrap().len(), 1); - - assert_eq!( - remove_subscription(&mut subscriptions, &SubscriptionId::Number(0)), - true - ); - assert_eq!(subscriptions.len(), num_keys as usize); - assert_eq!(subscriptions.get(&0).unwrap().len(), 1); - assert_eq!( - remove_subscription(&mut subscriptions, &SubscriptionId::Number(0)), - false - ); - - assert_eq!(remove_subscription(&mut subscriptions, &extra_sub_id), true); - assert_eq!(subscriptions.len(), (num_keys - 1) as usize); - assert!(subscriptions.get(&0).is_none()); + rpc.root_unsubscribe(sub_id).unwrap(); + subscriptions + .control + .assert_unsubscribed(&SubscriptionParams::Root); } #[test] @@ -2453,12 +1963,8 @@ pub(crate) mod tests { OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); let mut pending_optimistically_confirmed_banks = HashSet::new(); - let (subscriber0, _id_receiver, transport_receiver0) = - Subscriber::new_test("accountNotification"); - let (subscriber1, _id_receiver, transport_receiver1) = - Subscriber::new_test("accountNotification"); let exit = Arc::new(AtomicBool::new(false)); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests_with_slots( @@ -2466,24 +1972,20 @@ pub(crate) mod tests { ))), optimistically_confirmed_bank.clone(), )); - let sub_id0 = SubscriptionId::Number(0); - subscriptions.add_account_subscription( - alice.pubkey(), - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - encoding: None, - data_slice: None, - }), - sub_id0.clone(), - subscriber0, - ); + let (rpc0, mut receiver0) = rpc_pubsub_service::test_connection(&subscriptions); + let (rpc1, mut receiver1) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id0 = rpc0 + .account_subscribe( + alice.pubkey().to_string(), + Some(RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + encoding: None, + data_slice: None, + }), + ) + .unwrap(); - assert!(subscriptions - .subscriptions - .gossip_account_subscriptions - .read() - .unwrap() - .contains_key(&alice.pubkey())); + assert!(subscriptions.control.account_subscribed(&alice.pubkey())); let tx = system_transaction::create_account( &mint_keypair, @@ -2533,7 +2035,7 @@ pub(crate) mod tests { &mut highest_confirmed_slot, ); - let (response, _) = robust_poll_or_panic(transport_receiver0); + let response = receiver0.recv(); let expected = json!({ "jsonrpc": "2.0", "method": "accountNotification", @@ -2551,20 +2053,22 @@ pub(crate) mod tests { "subscription": 0, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&sub_id0); - - let sub_id1 = SubscriptionId::Number(1); - subscriptions.add_account_subscription( - alice.pubkey(), - Some(RpcAccountInfoConfig { - commitment: Some(CommitmentConfig::confirmed()), - encoding: None, - data_slice: None, - }), - sub_id1.clone(), - subscriber1, + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), ); + rpc0.account_unsubscribe(sub_id0).unwrap(); + + let sub_id1 = rpc1 + .account_subscribe( + alice.pubkey().to_string(), + Some(RpcAccountInfoConfig { + commitment: Some(CommitmentConfig::confirmed()), + encoding: None, + data_slice: None, + }), + ) + .unwrap(); let bank2 = bank_forks.read().unwrap().get(2).unwrap().clone(); bank2.freeze(); @@ -2578,7 +2082,7 @@ pub(crate) mod tests { &mut last_notified_confirmed_slot, &mut highest_confirmed_slot, ); - let (response, _) = robust_poll_or_panic(transport_receiver1); + let response = receiver1.recv(); let expected = json!({ "jsonrpc": "2.0", "method": "accountNotification", @@ -2596,60 +2100,13 @@ pub(crate) mod tests { "subscription": 1, } }); - assert_eq!(serde_json::to_string(&expected).unwrap(), response); - subscriptions.remove_account_subscription(&sub_id1); + assert_eq!( + expected, + serde_json::from_str::(&response).unwrap(), + ); + rpc1.account_unsubscribe(sub_id1).unwrap(); - assert!(!subscriptions - .subscriptions - .gossip_account_subscriptions - .read() - .unwrap() - .contains_key(&alice.pubkey())); - } - - #[test] - fn test_total_nested_subscriptions() { - let mock_subscriptions = RwLock::new(HashMap::new()); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 0); - - mock_subscriptions - .write() - .unwrap() - .insert(0, HashMap::new()); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 0); - - mock_subscriptions - .write() - .unwrap() - .entry(0) - .and_modify(|map| { - map.insert(0, "test"); - }); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 1); - - mock_subscriptions - .write() - .unwrap() - .entry(0) - .and_modify(|map| { - map.insert(1, "test"); - }); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 2); - - mock_subscriptions - .write() - .unwrap() - .insert(1, HashMap::new()); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 2); - - mock_subscriptions - .write() - .unwrap() - .entry(1) - .and_modify(|map| { - map.insert(0, "test"); - }); - assert_eq!(total_nested_subscriptions(&mock_subscriptions), 3); + assert!(!subscriptions.control.account_subscribed(&alice.pubkey())); } #[test] @@ -2657,99 +2114,79 @@ pub(crate) mod tests { let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100); let bank = Bank::new(&genesis_config); let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); - let subscriptions = RpcSubscriptions::default_with_bank_forks(bank_forks); + let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks)); + + let (rpc1, _receiver1) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id1 = rpc1 + .account_subscribe(Pubkey::default().to_string(), None) + .unwrap(); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("accountNotification"); - let account_sub_id = SubscriptionId::Number(0u64); - subscriptions.add_account_subscription( - Pubkey::default(), - None, - account_sub_id.clone(), - subscriber, - ); assert_eq!(subscriptions.total(), 1); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("programNotification"); - let program_sub_id = SubscriptionId::Number(1u64); - subscriptions.add_program_subscription( - Pubkey::default(), - None, - program_sub_id.clone(), - subscriber, - ); + let (rpc2, _receiver2) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id2 = rpc2 + .program_subscribe(Pubkey::default().to_string(), None) + .unwrap(); + assert_eq!(subscriptions.total(), 2); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("logsNotification"); - let logs_sub_id = SubscriptionId::Number(2u64); - subscriptions.add_logs_subscription(None, false, None, logs_sub_id.clone(), subscriber); + let (rpc3, _receiver3) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id3 = rpc3 + .logs_subscribe(RpcTransactionLogsFilter::All, None) + .unwrap(); assert_eq!(subscriptions.total(), 3); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("signatureNotification"); - let sig_sub_id = SubscriptionId::Number(3u64); - subscriptions.add_signature_subscription( - Signature::default(), - None, - sig_sub_id.clone(), - subscriber, - ); + let (rpc4, _receiver4) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id4 = rpc4 + .signature_subscribe(Signature::default().to_string(), None) + .unwrap(); + assert_eq!(subscriptions.total(), 4); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("slotNotification"); - let slot_sub_id = SubscriptionId::Number(4u64); - subscriptions.add_slot_subscription(slot_sub_id.clone(), subscriber); + let (rpc5, _receiver5) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id5 = rpc5.slot_subscribe().unwrap(); + assert_eq!(subscriptions.total(), 5); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("voteNotification"); - let vote_sub_id = SubscriptionId::Number(5u64); - subscriptions.add_vote_subscription(vote_sub_id.clone(), subscriber); + let (rpc6, _receiver6) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id6 = rpc6.vote_subscribe().unwrap(); + assert_eq!(subscriptions.total(), 6); - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("rootNotification"); - let root_sub_id = SubscriptionId::Number(6u64); - subscriptions.add_root_subscription(root_sub_id.clone(), subscriber); + let (rpc7, _receiver7) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id7 = rpc7.root_subscribe().unwrap(); + assert_eq!(subscriptions.total(), 7); - // Add duplicate account subscription to ensure totals include all subscriptions on all keys - let (subscriber, _id_receiver, _transport_receiver) = - Subscriber::new_test("accountNotification2"); - let account_dupe_sub_id = SubscriptionId::Number(7u64); - subscriptions.add_account_subscription( - Pubkey::default(), - None, - account_dupe_sub_id.clone(), - subscriber, - ); - assert_eq!(subscriptions.total(), 8); - - subscriptions.remove_account_subscription(&account_sub_id); + // Add duplicate account subscription, but it shouldn't increment the count. + let (rpc8, _receiver8) = rpc_pubsub_service::test_connection(&subscriptions); + let sub_id8 = rpc8 + .account_subscribe(Pubkey::default().to_string(), None) + .unwrap(); assert_eq!(subscriptions.total(), 7); - subscriptions.remove_account_subscription(&account_dupe_sub_id); + rpc1.account_unsubscribe(sub_id1).unwrap(); + assert_eq!(subscriptions.total(), 7); + + rpc8.account_unsubscribe(sub_id8).unwrap(); assert_eq!(subscriptions.total(), 6); - subscriptions.remove_program_subscription(&program_sub_id); + rpc2.program_unsubscribe(sub_id2).unwrap(); assert_eq!(subscriptions.total(), 5); - subscriptions.remove_logs_subscription(&logs_sub_id); + rpc3.logs_unsubscribe(sub_id3).unwrap(); assert_eq!(subscriptions.total(), 4); - subscriptions.remove_signature_subscription(&sig_sub_id); + rpc4.signature_unsubscribe(sub_id4).unwrap(); assert_eq!(subscriptions.total(), 3); - subscriptions.remove_slot_subscription(&slot_sub_id); + rpc5.slot_unsubscribe(sub_id5).unwrap(); assert_eq!(subscriptions.total(), 2); - subscriptions.remove_vote_subscription(&vote_sub_id); + rpc6.vote_unsubscribe(sub_id6).unwrap(); assert_eq!(subscriptions.total(), 1); - subscriptions.remove_root_subscription(&root_sub_id); + rpc7.root_unsubscribe(sub_id7).unwrap(); assert_eq!(subscriptions.total(), 0); } } diff --git a/core/src/tvu.rs b/core/src/tvu.rs index b0c5ba5a48..c6d4dd9ed6 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -108,7 +108,7 @@ impl Tvu { sockets: Sockets, blockstore: Arc, ledger_signal_receiver: Receiver, - subscriptions: &Arc, + rpc_subscriptions: &Arc, poh_recorder: &Arc>, tower: Tower, leader_schedule_cache: &Arc, @@ -187,7 +187,7 @@ impl Tvu { tvu_config.repair_validators, completed_data_sets_sender, max_slots, - Some(subscriptions.clone()), + Some(rpc_subscriptions.clone()), duplicate_slots_sender, ); @@ -255,7 +255,7 @@ impl Tvu { vote_account: *vote_account, authorized_voter_keypairs, exit: exit.clone(), - subscriptions: subscriptions.clone(), + rpc_subscriptions: rpc_subscriptions.clone(), leader_schedule_cache: leader_schedule_cache.clone(), latest_root_senders: vec![ledger_cleanup_slot_sender], accounts_background_request_sender, @@ -411,7 +411,7 @@ pub mod tests { }, blockstore, ledger_signal_receiver, - &Arc::new(RpcSubscriptions::new( + &Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks.clone(), block_commitment_cache.clone(), diff --git a/core/src/validator.rs b/core/src/validator.rs index d8320a9943..edac6531f3 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -471,12 +471,12 @@ impl Validator { let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new_with_vote_subscription( + let rpc_subscriptions = Arc::new(RpcSubscriptions::new_with_config( &exit, bank_forks.clone(), block_commitment_cache.clone(), optimistically_confirmed_bank.clone(), - config.pubsub_config.enable_vote_subscription, + &config.pubsub_config, )); let max_slots = Arc::new(MaxSlots::default()); @@ -485,7 +485,7 @@ impl Validator { let completed_data_sets_service = CompletedDataSetsService::new( completed_data_sets_receiver, blockstore.clone(), - subscriptions.clone(), + rpc_subscriptions.clone(), &exit, max_slots.clone(), ); @@ -563,19 +563,25 @@ impl Validator { if config.rpc_config.minimal_api { None } else { - Some(PubSubService::new( + let (trigger, pubsub_service) = PubSubService::new( config.pubsub_config.clone(), - &subscriptions, + &rpc_subscriptions, rpc_pubsub_addr, - &exit, - )) + ); + config + .validator_exit + .write() + .unwrap() + .register_exit(Box::new(move || trigger.cancel())); + + Some(pubsub_service) }, Some(OptimisticallyConfirmedBankTracker::new( bank_notification_receiver, &exit, bank_forks.clone(), optimistically_confirmed_bank, - subscriptions.clone(), + rpc_subscriptions.clone(), )), Some(bank_notification_sender), ) @@ -711,7 +717,7 @@ impl Validator { }, blockstore.clone(), ledger_signal_receiver, - &subscriptions, + &rpc_subscriptions, &poh_recorder, tower, &leader_schedule_cache, @@ -757,7 +763,7 @@ impl Validator { node.sockets.tpu, node.sockets.tpu_forwards, node.sockets.broadcast, - &subscriptions, + &rpc_subscriptions, transaction_status_sender, &blockstore, &config.broadcast_stage_type, diff --git a/core/tests/client.rs b/core/tests/client.rs index 7350647513..6f3df70b23 100644 --- a/core/tests/client.rs +++ b/core/tests/client.rs @@ -96,14 +96,14 @@ fn test_slot_subscription() { let bank_forks = Arc::new(RwLock::new(BankForks::new(bank))); let optimistically_confirmed_bank = OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); - let subscriptions = Arc::new(RpcSubscriptions::new( + let subscriptions = Arc::new(RpcSubscriptions::new_for_tests( &exit, bank_forks, Arc::new(RwLock::new(BlockCommitmentCache::default())), optimistically_confirmed_bank, )); - let pubsub_service = - PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr, &exit); + let (trigger, pubsub_service) = + PubSubService::new(PubSubConfig::default(), &subscriptions, pubsub_addr); std::thread::sleep(Duration::from_millis(400)); let (mut client, receiver) = @@ -136,6 +136,7 @@ fn test_slot_subscription() { } exit.store(true, Ordering::Relaxed); + trigger.cancel(); client.shutdown().unwrap(); pubsub_service.close().unwrap(); diff --git a/core/tests/rpc.rs b/core/tests/rpc.rs index a976f321d3..39b5501e74 100644 --- a/core/tests/rpc.rs +++ b/core/tests/rpc.rs @@ -8,7 +8,7 @@ use solana_account_decoder::UiAccount; use solana_client::{ rpc_client::RpcClient, rpc_config::{RpcAccountInfoConfig, RpcSignatureSubscribeConfig}, - rpc_response::{Response, RpcSignatureResult, SlotUpdate}, + rpc_response::{Response as RpcResponse, RpcSignatureResult, SlotUpdate}, tpu_client::{TpuClient, TpuClientConfig}, }; use solana_core::{rpc_pubsub::gen_client::Client as PubsubClient, test_validator::TestValidator}; @@ -243,9 +243,9 @@ fn test_rpc_subscriptions() { // Track when subscriptions are ready let (ready_sender, ready_receiver) = channel::<()>(); // Track account notifications are received - let (account_sender, account_receiver) = channel::>(); + let (account_sender, account_receiver) = channel::>(); // Track when status notifications are received - let (status_sender, status_receiver) = channel::<(String, Response)>(); + let (status_sender, status_receiver) = channel::<(String, RpcResponse)>(); // Create the pub sub runtime let rt = Runtime::new().unwrap(); diff --git a/metrics/src/lib.rs b/metrics/src/lib.rs index d0ca7264ce..01c03a4269 100644 --- a/metrics/src/lib.rs +++ b/metrics/src/lib.rs @@ -3,3 +3,61 @@ pub mod counter; pub mod datapoint; mod metrics; pub use crate::metrics::{flush, query, set_host_id, set_panic_hook, submit}; + +use std::sync::Arc; + +/// A helper that sends the count of created tokens as a datapoint. +#[allow(clippy::redundant_allocation)] +pub struct TokenCounter(Arc<&'static str>); + +impl TokenCounter { + /// Creates a new counter with the specified metrics `name`. + pub fn new(name: &'static str) -> Self { + Self(Arc::new(name)) + } + + /// Creates a new token for this counter. The metric's value will be equal + /// to the number of `CounterToken`s. + pub fn create_token(&self) -> CounterToken { + // new_count = strong_count + // - 1 (in TokenCounter) + // + 1 (token that's being created) + datapoint_info!(*self.0, ("count", Arc::strong_count(&self.0), i64)); + CounterToken(self.0.clone()) + } +} + +/// A token for `TokenCounter`. +#[allow(clippy::redundant_allocation)] +pub struct CounterToken(Arc<&'static str>); + +impl Clone for CounterToken { + fn clone(&self) -> Self { + // new_count = strong_count + // - 1 (in TokenCounter) + // + 1 (token that's being created) + datapoint_info!(*self.0, ("count", Arc::strong_count(&self.0), i64)); + CounterToken(self.0.clone()) + } +} + +impl Drop for CounterToken { + fn drop(&mut self) { + // new_count = strong_count + // - 1 (in TokenCounter, if it still exists) + // - 1 (token that's being dropped) + datapoint_info!( + *self.0, + ("count", Arc::strong_count(&self.0).saturating_sub(2), i64) + ); + } +} + +impl Drop for TokenCounter { + fn drop(&mut self) { + datapoint_info!( + *self.0, + ("count", Arc::strong_count(&self.0).saturating_sub(2), i64) + ); + } +} diff --git a/replica-node/src/replica_node.rs b/replica-node/src/replica_node.rs new file mode 100644 index 0000000000..a7a9d2d1a9 --- /dev/null +++ b/replica-node/src/replica_node.rs @@ -0,0 +1,339 @@ +use { + crate::accountsdb_repl_service::AccountsDbReplService, + crossbeam_channel::unbounded, + log::*, + solana_download_utils::download_snapshot_archive, + solana_genesis_utils::download_then_check_genesis_hash, + solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo}, + solana_ledger::{ + blockstore::Blockstore, blockstore_db::AccessType, blockstore_processor, + leader_schedule_cache::LeaderScheduleCache, + }, + solana_replica_lib::accountsdb_repl_client::AccountsDbReplClientServiceConfig, + solana_rpc::{ + max_slots::MaxSlots, + optimistically_confirmed_bank_tracker::{ + OptimisticallyConfirmedBank, OptimisticallyConfirmedBankTracker, + }, + rpc::JsonRpcConfig, + rpc_pubsub_service::{PubSubConfig, PubSubService}, + rpc_service::JsonRpcService, + rpc_subscriptions::RpcSubscriptions, + }, + solana_runtime::{ + accounts_index::AccountSecondaryIndexes, bank_forks::BankForks, + commitment::BlockCommitmentCache, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + snapshot_config::SnapshotConfig, snapshot_package::SnapshotType, snapshot_utils, + }, + solana_sdk::{clock::Slot, exit::Exit, genesis_config::GenesisConfig, hash::Hash}, + solana_streamer::socket::SocketAddrSpace, + std::{ + fs, + net::SocketAddr, + path::PathBuf, + sync::{ + atomic::{AtomicBool, AtomicU64}, + Arc, RwLock, + }, + }, +}; + +pub struct ReplicaNodeConfig { + pub rpc_peer_addr: SocketAddr, + pub accountsdb_repl_peer_addr: Option, + pub rpc_addr: SocketAddr, + pub rpc_pubsub_addr: SocketAddr, + pub ledger_path: PathBuf, + pub snapshot_archives_dir: PathBuf, + pub bank_snapshots_dir: PathBuf, + pub account_paths: Vec, + pub snapshot_info: (Slot, Hash), + pub cluster_info: Arc, + pub rpc_config: JsonRpcConfig, + pub snapshot_config: Option, + pub pubsub_config: PubSubConfig, + pub account_indexes: AccountSecondaryIndexes, + pub accounts_db_caching_enabled: bool, + pub replica_exit: Arc>, + pub socket_addr_space: SocketAddrSpace, +} + +pub struct ReplicaNode { + json_rpc_service: Option, + pubsub_service: Option, + optimistically_confirmed_bank_tracker: Option, + accountsdb_repl_service: Option, +} + +// Struct maintaining information about banks +struct ReplicaBankInfo { + bank_forks: Arc>, + optimistically_confirmed_bank: Arc>, + leader_schedule_cache: Arc, + block_commitment_cache: Arc>, +} + +// Initialize the replica by downloading snapshot from the peer, initialize +// the BankForks, OptimisticallyConfirmedBank, LeaderScheduleCache and +// BlockCommitmentCache and return the info wrapped as ReplicaBankInfo. +fn initialize_from_snapshot( + replica_config: &ReplicaNodeConfig, + snapshot_config: &SnapshotConfig, + genesis_config: &GenesisConfig, +) -> ReplicaBankInfo { + info!( + "Downloading snapshot from the peer into {:?}", + replica_config.snapshot_archives_dir + ); + + download_snapshot_archive( + &replica_config.rpc_peer_addr, + &replica_config.snapshot_archives_dir, + replica_config.snapshot_info, + SnapshotType::FullSnapshot, + snapshot_config.maximum_full_snapshot_archives_to_retain, + snapshot_config.maximum_incremental_snapshot_archives_to_retain, + false, + &mut None, + ) + .unwrap(); + + fs::create_dir_all(&snapshot_config.bank_snapshots_dir) + .expect("Couldn't create bank snapshot directory"); + + let archive_info = snapshot_utils::get_highest_full_snapshot_archive_info( + &replica_config.snapshot_archives_dir, + ) + .unwrap(); + + let process_options = blockstore_processor::ProcessOptions { + account_indexes: replica_config.account_indexes.clone(), + accounts_db_caching_enabled: replica_config.accounts_db_caching_enabled, + ..blockstore_processor::ProcessOptions::default() + }; + + info!( + "Build bank from snapshot archive: {:?}", + &snapshot_config.bank_snapshots_dir + ); + let (bank0, _) = snapshot_utils::bank_from_snapshot_archives( + &replica_config.account_paths, + &[], + &snapshot_config.bank_snapshots_dir, + &archive_info, + None, + genesis_config, + process_options.debug_keys.clone(), + None, + process_options.account_indexes.clone(), + process_options.accounts_db_caching_enabled, + process_options.limit_load_slot_count_from_snapshot, + process_options.shrink_ratio, + process_options.accounts_db_test_hash_calculation, + false, + process_options.verify_index, + process_options.accounts_db_config, + ) + .unwrap(); + + let bank0_slot = bank0.slot(); + let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank0)); + + let bank_forks = Arc::new(RwLock::new(BankForks::new(bank0))); + + let optimistically_confirmed_bank = + OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks); + + let mut block_commitment_cache = BlockCommitmentCache::default(); + block_commitment_cache.initialize_slots(bank0_slot); + let block_commitment_cache = Arc::new(RwLock::new(block_commitment_cache)); + + ReplicaBankInfo { + bank_forks, + optimistically_confirmed_bank, + leader_schedule_cache, + block_commitment_cache, + } +} + +fn start_client_rpc_services( + replica_config: &ReplicaNodeConfig, + genesis_config: &GenesisConfig, + cluster_info: Arc, + bank_info: &ReplicaBankInfo, + socket_addr_space: &SocketAddrSpace, +) -> ( + Option, + Option, + Option, +) { + let ReplicaBankInfo { + bank_forks, + optimistically_confirmed_bank, + leader_schedule_cache, + block_commitment_cache, + } = bank_info; + let blockstore = Arc::new( + Blockstore::open_with_access_type( + &replica_config.ledger_path, + AccessType::PrimaryOnly, + None, + false, + ) + .unwrap(), + ); + + let max_complete_transaction_status_slot = Arc::new(AtomicU64::new(0)); + + let max_slots = Arc::new(MaxSlots::default()); + let exit = Arc::new(AtomicBool::new(false)); + + let subscriptions = Arc::new(RpcSubscriptions::new( + &exit, + bank_forks.clone(), + block_commitment_cache.clone(), + optimistically_confirmed_bank.clone(), + )); + + let rpc_override_health_check = Arc::new(AtomicBool::new(false)); + if ContactInfo::is_valid_address(&replica_config.rpc_addr, socket_addr_space) { + assert!(ContactInfo::is_valid_address( + &replica_config.rpc_pubsub_addr, + socket_addr_space + )); + } else { + assert!(!ContactInfo::is_valid_address( + &replica_config.rpc_pubsub_addr, + socket_addr_space + )); + } + + let (trigger, pubsub_service) = PubSubService::new( + replica_config.pubsub_config.clone(), + &subscriptions, + replica_config.rpc_pubsub_addr, + ); + replica_config + .replica_exit + .write() + .unwrap() + .register_exit(Box::new(move || trigger.cancel())); + + let (_bank_notification_sender, bank_notification_receiver) = unbounded(); + ( + Some(JsonRpcService::new( + replica_config.rpc_addr, + replica_config.rpc_config.clone(), + replica_config.snapshot_config.clone(), + bank_forks.clone(), + block_commitment_cache.clone(), + blockstore, + cluster_info, + None, + genesis_config.hash(), + &replica_config.ledger_path, + replica_config.replica_exit.clone(), + None, + rpc_override_health_check, + optimistically_confirmed_bank.clone(), + 0, + 0, + max_slots, + leader_schedule_cache.clone(), + max_complete_transaction_status_slot, + )), + Some(pubsub_service), + Some(OptimisticallyConfirmedBankTracker::new( + bank_notification_receiver, + &exit, + bank_forks.clone(), + optimistically_confirmed_bank.clone(), + subscriptions, + None, + )), + ) +} + +impl ReplicaNode { + pub fn new(replica_config: ReplicaNodeConfig) -> Self { + let genesis_config = download_then_check_genesis_hash( + &replica_config.rpc_peer_addr, + &replica_config.ledger_path, + None, + MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + false, + true, + ) + .unwrap(); + + let snapshot_config = SnapshotConfig { + full_snapshot_archive_interval_slots: Slot::MAX, + incremental_snapshot_archive_interval_slots: Slot::MAX, + snapshot_archives_dir: replica_config.snapshot_archives_dir.clone(), + bank_snapshots_dir: replica_config.bank_snapshots_dir.clone(), + ..SnapshotConfig::default() + }; + + let bank_info = + initialize_from_snapshot(&replica_config, &snapshot_config, &genesis_config); + + let (json_rpc_service, pubsub_service, optimistically_confirmed_bank_tracker) = + start_client_rpc_services( + &replica_config, + &genesis_config, + replica_config.cluster_info.clone(), + &bank_info, + &replica_config.socket_addr_space, + ); + + let accountsdb_repl_client_config = AccountsDbReplClientServiceConfig { + worker_threads: 1, + replica_server_addr: replica_config.accountsdb_repl_peer_addr.unwrap(), + }; + + let last_replicated_slot = bank_info.bank_forks.read().unwrap().root_bank().slot(); + info!( + "Starting AccountsDbReplService from slot {:?}", + last_replicated_slot + ); + let accountsdb_repl_service = Some( + AccountsDbReplService::new(last_replicated_slot, accountsdb_repl_client_config) + .expect("Failed to start AccountsDb replication service"), + ); + + info!( + "Started AccountsDbReplService from slot {:?}", + last_replicated_slot + ); + + ReplicaNode { + json_rpc_service, + pubsub_service, + optimistically_confirmed_bank_tracker, + accountsdb_repl_service, + } + } + + pub fn join(self) { + if let Some(json_rpc_service) = self.json_rpc_service { + json_rpc_service.join().expect("rpc_service"); + } + + if let Some(pubsub_service) = self.pubsub_service { + pubsub_service.join().expect("pubsub_service"); + } + + if let Some(optimistically_confirmed_bank_tracker) = + self.optimistically_confirmed_bank_tracker + { + optimistically_confirmed_bank_tracker + .join() + .expect("optimistically_confirmed_bank_tracker"); + } + if let Some(accountsdb_repl_service) = self.accountsdb_repl_service { + accountsdb_repl_service + .join() + .expect("accountsdb_repl_service"); + } + } +} diff --git a/sdk/src/commitment_config.rs b/sdk/src/commitment_config.rs index 0551a068c0..1e77c65968 100644 --- a/sdk/src/commitment_config.rs +++ b/sdk/src/commitment_config.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use thiserror::Error; -#[derive(Serialize, Deserialize, Default, Clone, Copy, Debug, PartialEq)] +#[derive(Serialize, Deserialize, Default, Clone, Copy, Debug, PartialEq, Eq, Hash)] #[serde(rename_all = "camelCase")] pub struct CommitmentConfig { pub commitment: CommitmentLevel, diff --git a/validator/src/main.rs b/validator/src/main.rs index 6ecfb9acc1..ae2768f695 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1014,15 +1014,13 @@ pub fn main() { &format!("{}-{}", VALIDATOR_PORT_RANGE.0, VALIDATOR_PORT_RANGE.1); let default_genesis_archive_unpacked_size = &MAX_GENESIS_ARCHIVE_UNPACKED_SIZE.to_string(); let default_rpc_max_multiple_accounts = &MAX_MULTIPLE_ACCOUNTS.to_string(); - let default_rpc_pubsub_max_connections = PubSubConfig::default().max_connections.to_string(); - let default_rpc_pubsub_max_fragment_size = - PubSubConfig::default().max_fragment_size.to_string(); - let default_rpc_pubsub_max_in_buffer_capacity = - PubSubConfig::default().max_in_buffer_capacity.to_string(); - let default_rpc_pubsub_max_out_buffer_capacity = - PubSubConfig::default().max_out_buffer_capacity.to_string(); + let default_rpc_pubsub_max_active_subscriptions = PubSubConfig::default().max_active_subscriptions.to_string(); + let default_rpc_pubsub_queue_capacity_items = + PubSubConfig::default().queue_capacity_items.to_string(); + let default_rpc_pubsub_queue_capacity_bytes = + PubSubConfig::default().queue_capacity_bytes.to_string(); let default_rpc_send_transaction_retry_ms = ValidatorConfig::default() .send_transaction_retry_ms .to_string(); @@ -1558,10 +1556,10 @@ pub fn main() { .value_name("NUMBER") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_connections) + .hidden(true) .help("The maximum number of connections that RPC PubSub will support. \ This is a hard limit and no new connections beyond this limit can \ - be made until an old connection is dropped."), + be made until an old connection is dropped. (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_fragment_size") @@ -1569,9 +1567,9 @@ pub fn main() { .value_name("BYTES") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_fragment_size) + .hidden(true) .help("The maximum length in bytes of acceptable incoming frames. Messages longer \ - than this will be rejected."), + than this will be rejected. (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_in_buffer_capacity") @@ -1579,8 +1577,9 @@ pub fn main() { .value_name("BYTES") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_in_buffer_capacity) - .help("The maximum size in bytes to which the incoming websocket buffer can grow."), + .hidden(true) + .help("The maximum size in bytes to which the incoming websocket buffer can grow. \ + (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_out_buffer_capacity") @@ -1588,8 +1587,9 @@ pub fn main() { .value_name("BYTES") .takes_value(true) .validator(is_parsable::) - .default_value(&default_rpc_pubsub_max_out_buffer_capacity) - .help("The maximum size in bytes to which the outgoing websocket buffer can grow."), + .hidden(true) + .help("The maximum size in bytes to which the outgoing websocket buffer can grow. \ + (Obsolete)"), ) .arg( Arg::with_name("rpc_pubsub_max_active_subscriptions") @@ -1601,6 +1601,26 @@ pub fn main() { .help("The maximum number of active subscriptions that RPC PubSub will accept \ across all connections."), ) + .arg( + Arg::with_name("rpc_pubsub_queue_capacity_items") + .long("rpc-pubsub-queue-capacity-items") + .takes_value(true) + .value_name("NUMBER") + .validator(is_parsable::) + .default_value(&default_rpc_pubsub_queue_capacity_items) + .help("The maximum number of notifications that RPC PubSub will store \ + across all connections."), + ) + .arg( + Arg::with_name("rpc_pubsub_queue_capacity_bytes") + .long("rpc-pubsub-queue-capacity-bytes") + .takes_value(true) + .value_name("BYTES") + .validator(is_parsable::) + .default_value(&default_rpc_pubsub_queue_capacity_bytes) + .help("The maximum total size of notifications that RPC PubSub will store \ + across all connections."), + ) .arg( Arg::with_name("rpc_send_transaction_retry_ms") .long("rpc-send-retry-ms") @@ -2125,23 +2145,21 @@ pub fn main() { }), pubsub_config: PubSubConfig { enable_vote_subscription: matches.is_present("rpc_pubsub_enable_vote_subscription"), - max_connections: value_t_or_exit!(matches, "rpc_pubsub_max_connections", usize), - max_fragment_size: value_t_or_exit!(matches, "rpc_pubsub_max_fragment_size", usize), - max_in_buffer_capacity: value_t_or_exit!( - matches, - "rpc_pubsub_max_in_buffer_capacity", - usize - ), - max_out_buffer_capacity: value_t_or_exit!( - matches, - "rpc_pubsub_max_out_buffer_capacity", - usize - ), max_active_subscriptions: value_t_or_exit!( matches, "rpc_pubsub_max_active_subscriptions", usize ), + queue_capacity_items: value_t_or_exit!( + matches, + "rpc_pubsub_queue_capacity_items", + usize + ), + queue_capacity_bytes: value_t_or_exit!( + matches, + "rpc_pubsub_queue_capacity_bytes", + usize + ), }, voting_disabled: matches.is_present("no_voting") || restricted_repair_only_mode, wait_for_supermajority: value_t!(matches, "wait_for_supermajority", Slot).ok(),