gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add quic-client module (#23166)

Browse Source 
* Add quic-client module to send transactions via quic, abstracted behind the TpuConnection trait (along with a legacy UDP implementation of TpuConnection) and change thin-client to use TpuConnection
This commit is contained in:
ryleung-solana
2022-03-09 21:33:05 -05:00
committed by GitHub
parent 1fe0d6eeeb
commit 17b00ad3a4
18 changed files with 568 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
client/Cargo.toml
View File

@ -10,18 +10,24 @@ license = "Apache-2.0"
edition = "2021"
[dependencies]
async-mutex = "1.4.0"
async-trait = "0.1.52"
base64 = "0.13.0"
bincode = "1.3.3"
bs58 = "0.4.0"
bytes = "1.1.0"
clap = "2.33.0"
crossbeam-channel = "0.5"
futures = "0.3"
futures-util = "0.3.21"
indicatif = "0.16.2"
itertools = "0.10.2"
jsonrpc-core = "18.0.0"
log = "0.4.14"
quinn = "0.8.0"
rayon = "1.5.1"
reqwest = { version = "0.11.9", default-features = false, features = ["blocking", "rustls-tls", "json"] }
rustls = { version = "0.20.2", features = ["dangerous_configuration"] }
semver = "1.0.6"
serde = "1.0.136"
serde_derive = "1.0.103"

13
client/src/client_error.rs
View File

@ -1,6 +1,7 @@
pub use reqwest;
use {
crate::{rpc_request, rpc_response},
quinn::{ConnectError, WriteError},
solana_faucet::faucet::FaucetError,
solana_sdk::{
signature::SignerError, transaction::TransactionError, transport::TransportError,
@ -72,6 +73,18 @@ impl From<ClientErrorKind> for TransportError {
}
}
impl From<WriteError> for ClientErrorKind {
fn from(write_error: WriteError) -> Self {
Self::Custom(format!("{:?}", write_error))
}
}
impl From<ConnectError> for ClientErrorKind {
fn from(connect_error: ConnectError) -> Self {
Self::Custom(format!("{:?}", connect_error))
}
}
#[derive(Error, Debug)]
#[error("{kind}")]
pub struct ClientError {

3
client/src/lib.rs
View File

@ -10,6 +10,7 @@ pub mod nonblocking;
pub mod nonce_utils;
pub mod perf_utils;
pub mod pubsub_client;
pub mod quic_client;
pub mod rpc_cache;
pub mod rpc_client;
pub mod rpc_config;
@ -22,7 +23,9 @@ pub mod rpc_sender;
pub mod spinner;
pub mod thin_client;
pub mod tpu_client;
pub mod tpu_connection;
pub mod transaction_executor;
pub mod udp_client;
pub mod mock_sender_for_cli {
/// Magic `SIGNATURE` value used by `solana-cli` unit tests.

208
client/src/quic_client.rs Normal file
View File

@ -0,0 +1,208 @@
//! Simple client that connects to a given UDP port with the QUIC protocol and provides
//! an interface for sending transactions which is restricted by the server's flow control.
use {
crate::{client_error::ClientErrorKind, tpu_connection::TpuConnection},
async_mutex::Mutex,
futures::future::join_all,
itertools::Itertools,
quinn::{ClientConfig, Endpoint, EndpointConfig, NewConnection, WriteError},
rayon::iter::{IntoParallelIterator, ParallelIterator},
solana_sdk::{
quic::{QUIC_MAX_CONCURRENT_STREAMS, QUIC_PORT_OFFSET},
transaction::Transaction,
transport::Result as TransportResult,
},
std::{
net::{SocketAddr, UdpSocket},
sync::Arc,
},
tokio::runtime::Runtime,
};
struct SkipServerVerification;
impl SkipServerVerification {
pub fn new() -> Arc<Self> {
Arc::new(Self)
}
}
impl rustls::client::ServerCertVerifier for SkipServerVerification {
fn verify_server_cert(
&self,
_end_entity: &rustls::Certificate,
_intermediates: &[rustls::Certificate],
_server_name: &rustls::ServerName,
_scts: &mut dyn Iterator<Item = &[u8]>,
_ocsp_response: &[u8],
_now: std::time::SystemTime,
) -> Result<rustls::client::ServerCertVerified, rustls::Error> {
Ok(rustls::client::ServerCertVerified::assertion())
}
}
struct QuicClient {
runtime: Runtime,
endpoint: Endpoint,
connection: Arc<Mutex<Option<Arc<NewConnection>>>>,
addr: SocketAddr,
}
pub struct QuicTpuConnection {
client: Arc<QuicClient>,
}
impl TpuConnection for QuicTpuConnection {
fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self {
let tpu_addr = SocketAddr::new(tpu_addr.ip(), tpu_addr.port() + QUIC_PORT_OFFSET);
let client = Arc::new(QuicClient::new(client_socket, tpu_addr));
Self { client }
}
fn tpu_addr(&self) -> &SocketAddr {
&self.client.addr
}
fn send_wire_transaction(&self, data: Vec<u8>) -> TransportResult<()> {
let _guard = self.client.runtime.enter();
let send_buffer = self.client.send_buffer(&data[..]);
self.client.runtime.block_on(send_buffer)?;
Ok(())
}
fn send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
let buffers = transactions
.into_par_iter()
.map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch"))
.collect::<Vec<_>>();
let _guard = self.client.runtime.enter();
let send_batch = self.client.send_batch(&buffers[..]);
self.client.runtime.block_on(send_batch)?;
Ok(())
}
}
impl QuicClient {
pub fn new(client_socket: UdpSocket, addr: SocketAddr) -> Self {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let _guard = runtime.enter();
let crypto = rustls::ClientConfig::builder()
.with_safe_defaults()
.with_custom_certificate_verifier(SkipServerVerification::new())
.with_no_client_auth();
let create_endpoint = QuicClient::create_endpoint(EndpointConfig::default(), client_socket);
let mut endpoint = runtime.block_on(create_endpoint);
endpoint.set_default_client_config(ClientConfig::new(Arc::new(crypto)));
Self {
runtime,
endpoint,
connection: Arc::new(Mutex::new(None)),
addr,
}
}
// If this function becomes public, it should be changed to
// not expose details of the specific Quic implementation we're using
async fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint {
quinn::Endpoint::new(config, None, client_socket).unwrap().0
}
async fn _send_buffer_using_conn(
data: &[u8],
connection: &NewConnection,
) -> Result<(), WriteError> {
let mut send_stream = connection.connection.open_uni().await?;
send_stream.write_all(data).await?;
send_stream.finish().await?;
Ok(())
}
// Attempts to send data, connecting/reconnecting as necessary
// On success, returns the connection used to successfully send the data
async fn _send_buffer(&self, data: &[u8]) -> Result<Arc<NewConnection>, WriteError> {
let connection = {
let mut conn_guard = self.connection.lock().await;
let maybe_conn = (*conn_guard).clone();
match maybe_conn {
Some(conn) => conn.clone(),
None => {
let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
let connection = Arc::new(connecting.await?);
*conn_guard = Some(connection.clone());
connection
}
}
};
match Self::_send_buffer_using_conn(data, &connection).await {
Ok(()) => Ok(connection),
_ => {
let connection = {
let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
let connection = Arc::new(connecting.await?);
let mut conn_guard = self.connection.lock().await;
*conn_guard = Some(connection.clone());
connection
};
Self::_send_buffer_using_conn(data, &connection).await?;
Ok(connection)
}
}
}
pub async fn send_buffer(&self, data: &[u8]) -> Result<(), ClientErrorKind> {
self._send_buffer(data).await?;
Ok(())
}
pub async fn send_batch(&self, buffers: &[Vec<u8>]) -> Result<(), ClientErrorKind> {
// Start off by "testing" the connection by sending the first transaction
// This will also connect to the server if not already connected
// and reconnect and retry if the first send attempt failed
// (for example due to a timed out connection), returning an error
// or the connection that was used to successfully send the transaction.
// We will use the returned connection to send the rest of the transactions in the batch
// to avoid touching the mutex in self, and not bother reconnecting if we fail along the way
// since testing even in the ideal GCE environment has found no cases
// where reconnecting and retrying in the middle of a batch send
// (i.e. we encounter a connection error in the middle of a batch send, which presumably cannot
// be due to a timed out connection) has succeeded
if buffers.is_empty() {
return Ok(());
}
let connection = self._send_buffer(&buffers[0][..]).await?;
// Used to avoid dereferencing the Arc multiple times below
// by just getting a reference to the NewConnection once
let connection_ref: &NewConnection = &connection;
let chunks = buffers[1..buffers.len()]
.iter()
.chunks(QUIC_MAX_CONCURRENT_STREAMS);
let futures = chunks.into_iter().map(|buffs| {
join_all(
buffs
.into_iter()
.map(|buf| Self::_send_buffer_using_conn(&buf[..], connection_ref)),
)
});
for f in futures {
f.await.into_iter().try_for_each(|res| res)?;
}
Ok(())
}
}

88
client/src/thin_client.rs
View File

@ -4,8 +4,10 @@
//! unstable and may change in future releases.
use {
crate::{rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response},
bincode::{serialize_into, serialized_size},
crate::{
rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response,
tpu_connection::TpuConnection, udp_client::UdpTpuConnection,
},
log::*,
solana_sdk::{
account::Account,
@ -17,7 +19,6 @@ use {
hash::Hash,
instruction::Instruction,
message::Message,
packet::PACKET_DATA_SIZE,
pubkey::Pubkey,
signature::{Keypair, Signature, Signer},
signers::Signers,
@ -117,22 +118,20 @@ impl ClientOptimizer {
}
/// An object for querying and sending transactions to the network.
pub struct ThinClient {
transactions_socket: UdpSocket,
tpu_addrs: Vec<SocketAddr>,
pub struct ThinClient<C: 'static + TpuConnection> {
rpc_clients: Vec<RpcClient>,
tpu_connections: Vec<C>,
optimizer: ClientOptimizer,
}
impl ThinClient {
impl<C: 'static + TpuConnection> ThinClient<C> {
/// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP
/// and the Tpu at `tpu_addr` over `transactions_socket` using UDP.
/// and the Tpu at `tpu_addr` over `transactions_socket` using Quic or UDP
/// (currently hardcoded to UDP)
pub fn new(rpc_addr: SocketAddr, tpu_addr: SocketAddr, transactions_socket: UdpSocket) -> Self {
Self::new_from_client(
tpu_addr,
transactions_socket,
RpcClient::new_socket(rpc_addr),
)
let tpu_connection = C::new(transactions_socket, tpu_addr);
Self::new_from_client(RpcClient::new_socket(rpc_addr), tpu_connection)
}
pub fn new_socket_with_timeout(
@ -142,18 +141,14 @@ impl ThinClient {
timeout: Duration,
) -> Self {
let rpc_client = RpcClient::new_socket_with_timeout(rpc_addr, timeout);
Self::new_from_client(tpu_addr, transactions_socket, rpc_client)
let tpu_connection = C::new(transactions_socket, tpu_addr);
Self::new_from_client(rpc_client, tpu_connection)
}
fn new_from_client(
tpu_addr: SocketAddr,
transactions_socket: UdpSocket,
rpc_client: RpcClient,
) -> Self {
fn new_from_client(rpc_client: RpcClient, tpu_connection: C) -> Self {
Self {
transactions_socket,
tpu_addrs: vec![tpu_addr],
rpc_clients: vec![rpc_client],
tpu_connections: vec![tpu_connection],
optimizer: ClientOptimizer::new(0),
}
}
@ -168,16 +163,19 @@ impl ThinClient {
let rpc_clients: Vec<_> = rpc_addrs.into_iter().map(RpcClient::new_socket).collect();
let optimizer = ClientOptimizer::new(rpc_clients.len());
let tpu_connections: Vec<_> = tpu_addrs
.into_iter()
.map(|tpu_addr| C::new(transactions_socket.try_clone().unwrap(), tpu_addr))
.collect();
Self {
transactions_socket,
tpu_addrs,
rpc_clients,
tpu_connections,
optimizer,
}
}
fn tpu_addr(&self) -> &SocketAddr {
&self.tpu_addrs[self.optimizer.best()]
fn tpu_connection(&self) -> &C {
&self.tpu_connections[self.optimizer.best()]
}
fn rpc_client(&self) -> &RpcClient {
@ -205,7 +203,6 @@ impl ThinClient {
self.send_and_confirm_transaction(&[keypair], transaction, tries, 0)
}
/// Retry sending a signed Transaction to the server for processing
pub fn send_and_confirm_transaction<T: Signers>(
&self,
keypairs: &T,
@ -215,18 +212,13 @@ impl ThinClient {
) -> TransportResult<Signature> {
for x in 0..tries {
let now = Instant::now();
let mut buf = vec![0; serialized_size(&transaction).unwrap() as usize];
let mut wr = std::io::Cursor::new(&mut buf[..]);
let mut num_confirmed = 0;
let mut wait_time = MAX_PROCESSING_AGE;
serialize_into(&mut wr, &transaction)
.expect("serialize Transaction in pub fn transfer_signed");
// resend the same transaction until the transaction has no chance of succeeding
while now.elapsed().as_secs() < wait_time as u64 {
if num_confirmed == 0 {
// Send the transaction if there has been no confirmation (e.g. the first time)
self.transactions_socket
.send_to(&buf[..], &self.tpu_addr())?;
self.tpu_connection().send_transaction(transaction)?;
}
if let Ok(confirmed_blocks) = self.poll_for_signature_confirmation(
@ -321,13 +313,13 @@ impl ThinClient {
}
}
impl Client for ThinClient {
impl<C: 'static + TpuConnection> Client for ThinClient<C> {
fn tpu_addr(&self) -> String {
self.tpu_addr().to_string()
self.tpu_connection().tpu_addr().to_string()
}
}
impl SyncClient for ThinClient {
impl<C: 'static + TpuConnection> SyncClient for ThinClient<C> {
fn send_and_confirm_message<T: Signers>(
&self,
keypairs: &T,
@ -607,17 +599,16 @@ impl SyncClient for ThinClient {
}
}
impl AsyncClient for ThinClient {
impl<C: 'static + TpuConnection> AsyncClient for ThinClient<C> {
fn async_send_transaction(&self, transaction: Transaction) -> TransportResult<Signature> {
let mut buf = vec![0; serialized_size(&transaction).unwrap() as usize];
let mut wr = std::io::Cursor::new(&mut buf[..]);
serialize_into(&mut wr, &transaction)
.expect("serialize Transaction in pub fn transfer_signed");
assert!(buf.len() < PACKET_DATA_SIZE);
self.transactions_socket
.send_to(&buf[..], &self.tpu_addr())?;
self.tpu_connection().send_transaction(&transaction)?;
Ok(transaction.signatures[0])
}
fn async_send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
self.tpu_connection().send_batch(transactions)
}
fn async_send_message<T: Signers>(
&self,
keypairs: &T,
@ -649,20 +640,23 @@ impl AsyncClient for ThinClient {
}
}
pub fn create_client((rpc, tpu): (SocketAddr, SocketAddr), range: (u16, u16)) -> ThinClient {
pub fn create_client(
(rpc, tpu): (SocketAddr, SocketAddr),
range: (u16, u16),
) -> ThinClient<UdpTpuConnection> {
let (_, transactions_socket) =
solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap();
ThinClient::new(rpc, tpu, transactions_socket)
ThinClient::<UdpTpuConnection>::new(rpc, tpu, transactions_socket)
}
pub fn create_client_with_timeout(
(rpc, tpu): (SocketAddr, SocketAddr),
range: (u16, u16),
timeout: Duration,
) -> ThinClient {
) -> ThinClient<UdpTpuConnection> {
let (_, transactions_socket) =
solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap();
ThinClient::new_socket_with_timeout(rpc, tpu, transactions_socket, timeout)
ThinClient::<UdpTpuConnection>::new_socket_with_timeout(rpc, tpu, transactions_socket, timeout)
}
#[cfg(test)]

19
client/src/tpu_connection.rs Normal file
View File

@ -0,0 +1,19 @@
use {
solana_sdk::{transaction::Transaction, transport::Result as TransportResult},
std::net::{SocketAddr, UdpSocket},
};
pub trait TpuConnection {
fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self;
fn tpu_addr(&self) -> &SocketAddr;
fn send_transaction(&self, tx: &Transaction) -> TransportResult<()> {
let data = bincode::serialize(tx).expect("serialize Transaction in send_transaction");
self.send_wire_transaction(data)
}
fn send_wire_transaction(&self, data: Vec<u8>) -> TransportResult<()>;
fn send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()>;
}

42
client/src/udp_client.rs Normal file
View File

@ -0,0 +1,42 @@
//! Simple TPU client that communicates with the given UDP port with UDP and provides
//! an interface for sending transactions
use {
crate::tpu_connection::TpuConnection,
solana_sdk::{transaction::Transaction, transport::Result as TransportResult},
std::net::{SocketAddr, UdpSocket},
};
pub struct UdpTpuConnection {
socket: UdpSocket,
addr: SocketAddr,
}
impl TpuConnection for UdpTpuConnection {
fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self {
Self {
socket: client_socket,
addr: tpu_addr,
}
}
fn tpu_addr(&self) -> &SocketAddr {
&self.addr
}
fn send_wire_transaction(&self, data: Vec<u8>) -> TransportResult<()> {
self.socket.send_to(&data[..], self.addr)?;
Ok(())
}
fn send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
transactions
.into_iter()
.map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch"))
.try_for_each(|buff| -> TransportResult<()> {
self.socket.send_to(&buff[..], self.addr)?;
Ok(())
})?;
Ok(())
}
}