* ledger-storage-bigtable boilerplate (cherry picked from commit9d2293bb32
) * $ wget https://pki.goog/roots.pem -O pki-goog-roots.pem (cherry picked from commit1617a025ce
) * Add access_token module (cherry picked from commit59d266a111
) * Add root_ca_certificate (cherry picked from commitfaa016e4b7
) * Add build-proto (cherry picked from commitc31e1f5bf0
) * UiTransactionEncoding is now copy (cherry picked from commit494968be66
) * Increase timeout (cherry picked from commit57dfebc5ba
) * Add build-proto/build.sh output (cherry picked from commit54dae6ba2c
) * Supress doctest errors (cherry picked from commit019c75797d
) * Add compression (cherry picked from commit243e05d59f
) * Add bigtable (cherry picked from commit6e0353965a
) * Add configuration info (cherry picked from commit98cca1e774
) * Add ledger-tool bigtable subcommands (cherry picked from commitf9049d6ee4
) # Conflicts: # ledger-tool/Cargo.toml * Make room for tokio 0.2 (cherry picked from commitb876fb84ba
) # Conflicts: # core/Cargo.toml * Setup a tokio 0.2 runtime for RPC usage (cherry picked from commit0e02740565
) # Conflicts: # core/Cargo.toml * Plumb Bigtable ledger storage into the RPC subsystem (cherry picked from commitdfae9a9864
) # Conflicts: # core/Cargo.toml * Add RPC transaction history design (cherry picked from commite56ea138c7
) * Simplify access token refreshing (cherry picked from commit1f7af14386
) * Report block status more frequently (cherry picked from commit22c46ebf96
) * after -> before (cherry picked from commit227ea934ff
) * Rebase * Cargo.lock Co-authored-by: Michael Vines <mvines@gmail.com>
This commit is contained in:
113
core/src/rpc.rs
113
core/src/rpc.rs
@@ -67,6 +67,7 @@ use std::{
|
||||
Arc, Mutex, RwLock,
|
||||
},
|
||||
};
|
||||
use tokio::runtime;
|
||||
|
||||
fn new_response<T>(bank: &Bank, value: T) -> RpcResponse<T> {
|
||||
let context = RpcResponseContext { slot: bank.slot() };
|
||||
@@ -91,6 +92,7 @@ pub struct JsonRpcConfig {
|
||||
pub identity_pubkey: Pubkey,
|
||||
pub faucet_addr: Option<SocketAddr>,
|
||||
pub health_check_slot_distance: u64,
|
||||
pub enable_bigtable_ledger_storage: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -104,6 +106,8 @@ pub struct JsonRpcRequestProcessor {
|
||||
cluster_info: Arc<ClusterInfo>,
|
||||
genesis_hash: Hash,
|
||||
transaction_sender: Arc<Mutex<Sender<TransactionInfo>>>,
|
||||
runtime_handle: runtime::Handle,
|
||||
bigtable_ledger_storage: Option<solana_storage_bigtable::LedgerStorage>,
|
||||
}
|
||||
impl Metadata for JsonRpcRequestProcessor {}
|
||||
|
||||
@@ -158,6 +162,7 @@ impl JsonRpcRequestProcessor {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
config: JsonRpcConfig,
|
||||
bank_forks: Arc<RwLock<BankForks>>,
|
||||
@@ -167,6 +172,8 @@ impl JsonRpcRequestProcessor {
|
||||
health: Arc<RpcHealth>,
|
||||
cluster_info: Arc<ClusterInfo>,
|
||||
genesis_hash: Hash,
|
||||
runtime: &runtime::Runtime,
|
||||
bigtable_ledger_storage: Option<solana_storage_bigtable::LedgerStorage>,
|
||||
) -> (Self, Receiver<TransactionInfo>) {
|
||||
let (sender, receiver) = channel();
|
||||
(
|
||||
@@ -180,6 +187,8 @@ impl JsonRpcRequestProcessor {
|
||||
cluster_info,
|
||||
genesis_hash,
|
||||
transaction_sender: Arc::new(Mutex::new(sender)),
|
||||
runtime_handle: runtime.handle().clone(),
|
||||
bigtable_ledger_storage,
|
||||
},
|
||||
receiver,
|
||||
)
|
||||
@@ -218,6 +227,8 @@ impl JsonRpcRequestProcessor {
|
||||
cluster_info,
|
||||
genesis_hash,
|
||||
transaction_sender: Arc::new(Mutex::new(sender)),
|
||||
runtime_handle: runtime::Runtime::new().unwrap().handle().clone(),
|
||||
bigtable_ledger_storage: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -557,6 +568,7 @@ impl JsonRpcRequestProcessor {
|
||||
slot: Slot,
|
||||
encoding: Option<UiTransactionEncoding>,
|
||||
) -> Result<Option<ConfirmedBlock>> {
|
||||
let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
|
||||
if self.config.enable_rpc_transaction_history
|
||||
&& slot
|
||||
<= self
|
||||
@@ -565,7 +577,15 @@ impl JsonRpcRequestProcessor {
|
||||
.unwrap()
|
||||
.highest_confirmed_root()
|
||||
{
|
||||
let result = self.blockstore.get_confirmed_block(slot, encoding);
|
||||
let result = self.blockstore.get_confirmed_block(slot, Some(encoding));
|
||||
if result.is_err() {
|
||||
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
|
||||
return Ok(self
|
||||
.runtime_handle
|
||||
.block_on(bigtable_ledger_storage.get_confirmed_block(slot, encoding))
|
||||
.ok());
|
||||
}
|
||||
}
|
||||
self.check_slot_cleaned_up(&result, slot)?;
|
||||
Ok(result.ok())
|
||||
} else {
|
||||
@@ -594,9 +614,25 @@ impl JsonRpcRequestProcessor {
|
||||
MAX_GET_CONFIRMED_BLOCKS_RANGE
|
||||
)));
|
||||
}
|
||||
|
||||
let lowest_slot = self.blockstore.lowest_slot();
|
||||
if start_slot < lowest_slot {
|
||||
// If the starting slot is lower than what's available in blockstore assume the entire
|
||||
// [start_slot..end_slot] can be fetched from BigTable.
|
||||
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
|
||||
return Ok(self
|
||||
.runtime_handle
|
||||
.block_on(
|
||||
bigtable_ledger_storage
|
||||
.get_confirmed_blocks(start_slot, (end_slot - start_slot) as usize),
|
||||
)
|
||||
.unwrap_or_else(|_| vec![]));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.blockstore
|
||||
.rooted_slot_iterator(max(start_slot, self.blockstore.lowest_slot()))
|
||||
.rooted_slot_iterator(max(start_slot, lowest_slot))
|
||||
.map_err(|_| Error::internal_error())?
|
||||
.filter(|&slot| slot <= end_slot)
|
||||
.collect())
|
||||
@@ -690,6 +726,16 @@ impl JsonRpcRequestProcessor {
|
||||
err,
|
||||
}
|
||||
})
|
||||
.or_else(|| {
|
||||
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
|
||||
self.runtime_handle
|
||||
.block_on(bigtable_ledger_storage.get_signature_status(&signature))
|
||||
.map(Some)
|
||||
.unwrap_or(None)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -732,21 +778,38 @@ impl JsonRpcRequestProcessor {
|
||||
signature: Signature,
|
||||
encoding: Option<UiTransactionEncoding>,
|
||||
) -> Option<ConfirmedTransaction> {
|
||||
let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
|
||||
if self.config.enable_rpc_transaction_history {
|
||||
self.blockstore
|
||||
.get_confirmed_transaction(signature, encoding)
|
||||
match self
|
||||
.blockstore
|
||||
.get_confirmed_transaction(signature, Some(encoding))
|
||||
.unwrap_or(None)
|
||||
.filter(|confirmed_transaction| {
|
||||
confirmed_transaction.slot
|
||||
{
|
||||
Some(confirmed_transaction) => {
|
||||
if confirmed_transaction.slot
|
||||
<= self
|
||||
.block_commitment_cache
|
||||
.read()
|
||||
.unwrap()
|
||||
.highest_confirmed_root()
|
||||
})
|
||||
} else {
|
||||
None
|
||||
{
|
||||
return Some(confirmed_transaction);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
|
||||
return self
|
||||
.runtime_handle
|
||||
.block_on(
|
||||
bigtable_ledger_storage
|
||||
.get_confirmed_transaction(&signature, encoding),
|
||||
)
|
||||
.unwrap_or(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn get_confirmed_signatures_for_address(
|
||||
@@ -756,6 +819,8 @@ impl JsonRpcRequestProcessor {
|
||||
end_slot: Slot,
|
||||
) -> Vec<Signature> {
|
||||
if self.config.enable_rpc_transaction_history {
|
||||
// TODO: Add bigtable_ledger_storage support as a part of
|
||||
// https://github.com/solana-labs/solana/pull/10928
|
||||
let end_slot = min(
|
||||
end_slot,
|
||||
self.block_commitment_cache
|
||||
@@ -801,9 +866,23 @@ impl JsonRpcRequestProcessor {
|
||||
}
|
||||
|
||||
pub fn get_first_available_block(&self) -> Slot {
|
||||
self.blockstore
|
||||
let slot = self
|
||||
.blockstore
|
||||
.get_first_available_block()
|
||||
.unwrap_or_default()
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
|
||||
let bigtable_slot = self
|
||||
.runtime_handle
|
||||
.block_on(bigtable_ledger_storage.get_first_available_block())
|
||||
.unwrap_or(None)
|
||||
.unwrap_or(slot);
|
||||
|
||||
if bigtable_slot < slot {
|
||||
return bigtable_slot;
|
||||
}
|
||||
}
|
||||
slot
|
||||
}
|
||||
|
||||
pub fn get_stake_activation(
|
||||
@@ -2410,6 +2489,8 @@ pub mod tests {
|
||||
RpcHealth::stub(),
|
||||
cluster_info.clone(),
|
||||
Hash::default(),
|
||||
&runtime::Runtime::new().unwrap(),
|
||||
None,
|
||||
);
|
||||
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
|
||||
|
||||
@@ -3552,6 +3633,8 @@ pub mod tests {
|
||||
RpcHealth::stub(),
|
||||
cluster_info,
|
||||
Hash::default(),
|
||||
&runtime::Runtime::new().unwrap(),
|
||||
None,
|
||||
);
|
||||
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
|
||||
|
||||
@@ -3591,6 +3674,8 @@ pub mod tests {
|
||||
health.clone(),
|
||||
cluster_info,
|
||||
Hash::default(),
|
||||
&runtime::Runtime::new().unwrap(),
|
||||
None,
|
||||
);
|
||||
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
|
||||
|
||||
@@ -3771,6 +3856,8 @@ pub mod tests {
|
||||
RpcHealth::stub(),
|
||||
cluster_info,
|
||||
Hash::default(),
|
||||
&runtime::Runtime::new().unwrap(),
|
||||
None,
|
||||
);
|
||||
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
|
||||
assert_eq!(request_processor.validator_exit(), false);
|
||||
@@ -3798,6 +3885,8 @@ pub mod tests {
|
||||
RpcHealth::stub(),
|
||||
cluster_info,
|
||||
Hash::default(),
|
||||
&runtime::Runtime::new().unwrap(),
|
||||
None,
|
||||
);
|
||||
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
|
||||
assert_eq!(request_processor.validator_exit(), true);
|
||||
@@ -3887,6 +3976,8 @@ pub mod tests {
|
||||
RpcHealth::stub(),
|
||||
cluster_info,
|
||||
Hash::default(),
|
||||
&runtime::Runtime::new().unwrap(),
|
||||
None,
|
||||
);
|
||||
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
|
||||
assert_eq!(
|
||||
|
Reference in New Issue
Block a user