gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Long-term ledger storage with BigTable (bp #11222) (#11392)

Browse Source 
* ledger-storage-bigtable boilerplate

(cherry picked from commit 9d2293bb32)

* $ wget https://pki.goog/roots.pem -O pki-goog-roots.pem

(cherry picked from commit 1617a025ce)

* Add access_token module

(cherry picked from commit 59d266a111)

* Add root_ca_certificate

(cherry picked from commit faa016e4b7)

* Add build-proto

(cherry picked from commit c31e1f5bf0)

* UiTransactionEncoding is now copy

(cherry picked from commit 494968be66)

* Increase timeout

(cherry picked from commit 57dfebc5ba)

* Add build-proto/build.sh output

(cherry picked from commit 54dae6ba2c)

* Supress doctest errors

(cherry picked from commit 019c75797d)

* Add compression

(cherry picked from commit 243e05d59f)

* Add bigtable

(cherry picked from commit 6e0353965a)

* Add configuration info

(cherry picked from commit 98cca1e774)

* Add ledger-tool bigtable subcommands

(cherry picked from commit f9049d6ee4)

# Conflicts:
#	ledger-tool/Cargo.toml

* Make room for tokio 0.2

(cherry picked from commit b876fb84ba)

# Conflicts:
#	core/Cargo.toml

* Setup a tokio 0.2 runtime for RPC usage

(cherry picked from commit 0e02740565)

# Conflicts:
#	core/Cargo.toml

* Plumb Bigtable ledger storage into the RPC subsystem

(cherry picked from commit dfae9a9864)

# Conflicts:
#	core/Cargo.toml

* Add RPC transaction history design

(cherry picked from commit e56ea138c7)

* Simplify access token refreshing

(cherry picked from commit 1f7af14386)

* Report block status more frequently

(cherry picked from commit 22c46ebf96)

* after -> before

(cherry picked from commit 227ea934ff)

* Rebase

* Cargo.lock

Co-authored-by: Michael Vines <mvines@gmail.com>
This commit is contained in:
mergify[bot]
2020-08-06 04:06:44 +00:00
committed by GitHub
parent 6542a04521
commit 5841e4d665
35 changed files with 6214 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
core/src/rpc.rs
View File

@@ -67,6 +67,7 @@ use std::{
Arc, Mutex, RwLock,
},
};
use tokio::runtime;
fn new_response<T>(bank: &Bank, value: T) -> RpcResponse<T> {
let context = RpcResponseContext { slot: bank.slot() };
@@ -91,6 +92,7 @@ pub struct JsonRpcConfig {
pub identity_pubkey: Pubkey,
pub faucet_addr: Option<SocketAddr>,
pub health_check_slot_distance: u64,
pub enable_bigtable_ledger_storage: bool,
}
#[derive(Clone)]
@@ -104,6 +106,8 @@ pub struct JsonRpcRequestProcessor {
cluster_info: Arc<ClusterInfo>,
genesis_hash: Hash,
transaction_sender: Arc<Mutex<Sender<TransactionInfo>>>,
runtime_handle: runtime::Handle,
bigtable_ledger_storage: Option<solana_storage_bigtable::LedgerStorage>,
}
impl Metadata for JsonRpcRequestProcessor {}
@@ -158,6 +162,7 @@ impl JsonRpcRequestProcessor {
})
}
#[allow(clippy::too_many_arguments)]
pub fn new(
config: JsonRpcConfig,
bank_forks: Arc<RwLock<BankForks>>,
@@ -167,6 +172,8 @@ impl JsonRpcRequestProcessor {
health: Arc<RpcHealth>,
cluster_info: Arc<ClusterInfo>,
genesis_hash: Hash,
runtime: &runtime::Runtime,
bigtable_ledger_storage: Option<solana_storage_bigtable::LedgerStorage>,
) -> (Self, Receiver<TransactionInfo>) {
let (sender, receiver) = channel();
(
@@ -180,6 +187,8 @@ impl JsonRpcRequestProcessor {
cluster_info,
genesis_hash,
transaction_sender: Arc::new(Mutex::new(sender)),
runtime_handle: runtime.handle().clone(),
bigtable_ledger_storage,
},
receiver,
)
@@ -218,6 +227,8 @@ impl JsonRpcRequestProcessor {
cluster_info,
genesis_hash,
transaction_sender: Arc::new(Mutex::new(sender)),
runtime_handle: runtime::Runtime::new().unwrap().handle().clone(),
bigtable_ledger_storage: None,
}
}
@@ -557,6 +568,7 @@ impl JsonRpcRequestProcessor {
slot: Slot,
encoding: Option<UiTransactionEncoding>,
) -> Result<Option<ConfirmedBlock>> {
let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
if self.config.enable_rpc_transaction_history
&& slot
<= self
@@ -565,7 +577,15 @@ impl JsonRpcRequestProcessor {
.unwrap()
.highest_confirmed_root()
{
let result = self.blockstore.get_confirmed_block(slot, encoding);
let result = self.blockstore.get_confirmed_block(slot, Some(encoding));
if result.is_err() {
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
return Ok(self
.runtime_handle
.block_on(bigtable_ledger_storage.get_confirmed_block(slot, encoding))
.ok());
}
}
self.check_slot_cleaned_up(&result, slot)?;
Ok(result.ok())
} else {
@@ -594,9 +614,25 @@ impl JsonRpcRequestProcessor {
MAX_GET_CONFIRMED_BLOCKS_RANGE
)));
}
let lowest_slot = self.blockstore.lowest_slot();
if start_slot < lowest_slot {
// If the starting slot is lower than what's available in blockstore assume the entire
// [start_slot..end_slot] can be fetched from BigTable.
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
return Ok(self
.runtime_handle
.block_on(
bigtable_ledger_storage
.get_confirmed_blocks(start_slot, (end_slot - start_slot) as usize),
)
.unwrap_or_else(|_| vec![]));
}
}
Ok(self
.blockstore
.rooted_slot_iterator(max(start_slot, self.blockstore.lowest_slot()))
.rooted_slot_iterator(max(start_slot, lowest_slot))
.map_err(|_| Error::internal_error())?
.filter(|&slot| slot <= end_slot)
.collect())
@@ -690,6 +726,16 @@ impl JsonRpcRequestProcessor {
err,
}
})
.or_else(|| {
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
self.runtime_handle
.block_on(bigtable_ledger_storage.get_signature_status(&signature))
.map(Some)
.unwrap_or(None)
} else {
None
}
})
} else {
None
};
@@ -732,21 +778,38 @@ impl JsonRpcRequestProcessor {
signature: Signature,
encoding: Option<UiTransactionEncoding>,
) -> Option<ConfirmedTransaction> {
let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
if self.config.enable_rpc_transaction_history {
self.blockstore
.get_confirmed_transaction(signature, encoding)
match self
.blockstore
.get_confirmed_transaction(signature, Some(encoding))
.unwrap_or(None)
.filter(|confirmed_transaction| {
confirmed_transaction.slot
{
Some(confirmed_transaction) => {
if confirmed_transaction.slot
<= self
.block_commitment_cache
.read()
.unwrap()
.highest_confirmed_root()
})
} else {
None
{
return Some(confirmed_transaction);
}
}
None => {
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
return self
.runtime_handle
.block_on(
bigtable_ledger_storage
.get_confirmed_transaction(&signature, encoding),
)
.unwrap_or(None);
}
}
}
}
None
}
pub fn get_confirmed_signatures_for_address(
@@ -756,6 +819,8 @@ impl JsonRpcRequestProcessor {
end_slot: Slot,
) -> Vec<Signature> {
if self.config.enable_rpc_transaction_history {
// TODO: Add bigtable_ledger_storage support as a part of
// https://github.com/solana-labs/solana/pull/10928
let end_slot = min(
end_slot,
self.block_commitment_cache
@@ -801,9 +866,23 @@ impl JsonRpcRequestProcessor {
}
pub fn get_first_available_block(&self) -> Slot {
self.blockstore
let slot = self
.blockstore
.get_first_available_block()
.unwrap_or_default()
.unwrap_or_default();
if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
let bigtable_slot = self
.runtime_handle
.block_on(bigtable_ledger_storage.get_first_available_block())
.unwrap_or(None)
.unwrap_or(slot);
if bigtable_slot < slot {
return bigtable_slot;
}
}
slot
}
pub fn get_stake_activation(
@@ -2410,6 +2489,8 @@ pub mod tests {
RpcHealth::stub(),
cluster_info.clone(),
Hash::default(),
&runtime::Runtime::new().unwrap(),
None,
);
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
@@ -3552,6 +3633,8 @@ pub mod tests {
RpcHealth::stub(),
cluster_info,
Hash::default(),
&runtime::Runtime::new().unwrap(),
None,
);
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
@@ -3591,6 +3674,8 @@ pub mod tests {
health.clone(),
cluster_info,
Hash::default(),
&runtime::Runtime::new().unwrap(),
None,
);
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
@@ -3771,6 +3856,8 @@ pub mod tests {
RpcHealth::stub(),
cluster_info,
Hash::default(),
&runtime::Runtime::new().unwrap(),
None,
);
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
assert_eq!(request_processor.validator_exit(), false);
@@ -3798,6 +3885,8 @@ pub mod tests {
RpcHealth::stub(),
cluster_info,
Hash::default(),
&runtime::Runtime::new().unwrap(),
None,
);
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
assert_eq!(request_processor.validator_exit(), true);
@@ -3887,6 +3976,8 @@ pub mod tests {
RpcHealth::stub(),
cluster_info,
Hash::default(),
&runtime::Runtime::new().unwrap(),
None,
);
SendTransactionService::new(tpu_address, &bank_forks, &exit, receiver);
assert_eq!(

36
core/src/rpc_service.rs
View File

@@ -23,7 +23,7 @@ use std::{
sync::{mpsc::channel, Arc, RwLock},
thread::{self, Builder, JoinHandle},
};
use tokio::prelude::Future;
use tokio::runtime;
pub struct JsonRpcService {
thread_hdl: JoinHandle<()>,
@@ -32,6 +32,7 @@ pub struct JsonRpcService {
pub request_processor: JsonRpcRequestProcessor, // Used only by test_rpc_new()...
close_handle: Option<CloseHandle>,
runtime: runtime::Runtime,
}
struct RpcRequestMiddleware {
@@ -97,6 +98,9 @@ impl RpcRequestMiddleware {
}
fn process_file_get(&self, path: &str) -> RequestMiddlewareAction {
// Stuck on tokio 0.1 until the jsonrpc-http-server crate upgrades to tokio 0.2
use tokio_01::prelude::*;
let stem = path.split_at(1).1; // Drop leading '/' from path
let filename = {
match path {
@@ -115,10 +119,10 @@ impl RpcRequestMiddleware {
RequestMiddlewareAction::Respond {
should_validate_hosts: true,
response: Box::new(
tokio_fs::file::File::open(filename)
tokio_fs_01::file::File::open(filename)
.and_then(|file| {
let buf: Vec<u8> = Vec::new();
tokio_io::io::read_to_end(file, buf)
tokio_io_01::io::read_to_end(file, buf)
.and_then(|item| Ok(hyper::Response::new(item.1.into())))
.or_else(|_| Ok(RpcRequestMiddleware::internal_server_error()))
})
@@ -249,6 +253,28 @@ impl JsonRpcService {
));
let tpu_address = cluster_info.my_contact_info().tpu;
let mut runtime = runtime::Builder::new()
.threaded_scheduler()
.thread_name("rpc-runtime")
.enable_all()
.build()
.expect("Runtime");
let bigtable_ledger_storage = if config.enable_bigtable_ledger_storage {
runtime
.block_on(solana_storage_bigtable::LedgerStorage::new(false))
.map(|x| {
info!("BigTable ledger storage initialized");
Some(x)
})
.unwrap_or_else(|err| {
error!("Failed to initialize BigTable ledger storage: {:?}", err);
None
})
} else {
None
};
let (request_processor, receiver) = JsonRpcRequestProcessor::new(
config,
bank_forks.clone(),
@@ -258,6 +284,8 @@ impl JsonRpcService {
health.clone(),
cluster_info,
genesis_hash,
&runtime,
bigtable_ledger_storage,
);
let exit_send_transaction_service = Arc::new(AtomicBool::new(false));
@@ -325,6 +353,7 @@ impl JsonRpcService {
.register_exit(Box::new(move || close_handle_.close()));
Self {
thread_hdl,
runtime,
#[cfg(test)]
request_processor: test_request_processor,
close_handle: Some(close_handle),
@@ -338,6 +367,7 @@ impl JsonRpcService {
}
pub fn join(self) -> thread::Result<()> {
self.runtime.shutdown_background();
self.thread_hdl.join()
}
}

6
core/src/rpc_subscriptions.rs
View File

@@ -38,7 +38,9 @@ use std::{
iter,
sync::{Arc, Mutex, RwLock},
};
use tokio::runtime::{Builder as RuntimeBuilder, Runtime, TaskExecutor};
// Stuck on tokio 0.1 until the jsonrpc-pubsub crate upgrades to tokio 0.2
use tokio_01::runtime::{Builder as RuntimeBuilder, Runtime, TaskExecutor};
const RECEIVE_DELAY_MILLIS: u64 = 100;
@@ -950,7 +952,7 @@ pub(crate) mod tests {
system_transaction,
};
use std::{fmt::Debug, sync::mpsc::channel, time::Instant};
use tokio::{prelude::FutureExt, runtime::Runtime, timer::Delay};
use tokio_01::{prelude::FutureExt, runtime::Runtime, timer::Delay};
pub(crate) fn robust_poll_or_panic<T: Debug + Send + 'static>(
receiver: futures::sync::mpsc::Receiver<T>,