gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Stream additional block metadata via plugin (#22023)

Browse Source 
* Stream additional block metadata through plugin
blockhash, block_height, block_time, rewards are streamed
This commit is contained in:
Lijun Wang
2021-12-29 15:12:01 -08:00
committed by GitHub
parent c9c78622a8
commit f14928a970
13 changed files with 383 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs
View File

@ -12,7 +12,7 @@ use {
serde_json,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions,
ReplicaTransactionInfoVersions, Result, SlotStatus,
ReplicaBlockInfoVersions, ReplicaTransactionInfoVersions, Result, SlotStatus,
},
solana_metrics::*,
std::{fs::File, io::Read},
@ -334,6 +334,31 @@ impl AccountsDbPlugin for AccountsDbPluginPostgres {
Ok(())
}
fn notify_block_metadata(&mut self, block_info: ReplicaBlockInfoVersions) -> Result<()> {
match &mut self.client {
None => {
return Err(AccountsDbPluginError::Custom(Box::new(
AccountsDbPluginPostgresError::DataStoreConnectionError {
msg: "There is no connection to the PostgreSQL database.".to_string(),
},
)));
}
Some(client) => match block_info {
ReplicaBlockInfoVersions::V0_0_1(block_info) => {
let result = client.update_block_metadata(block_info);
if let Err(err) = result {
return Err(AccountsDbPluginError::SlotStatusUpdateError{
msg: format!("Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}", err)
});
}
}
},
}
Ok(())
}
/// Check if the plugin is interested in account data
/// Default is true -- if the plugin is not interested in
/// account data, please return false.

53
accountsdb-plugin-postgres/src/postgres_client.rs
View File

@ -1,4 +1,6 @@
#![allow(clippy::integer_arithmetic)]
mod postgres_client_block_metadata;
mod postgres_client_transaction;
/// A concurrent implementation for writing accounts into the PostgreSQL in parallel.
@ -10,9 +12,10 @@ use {
crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender},
log::*,
postgres::{Client, NoTls, Statement},
postgres_client_block_metadata::DbBlockInfo,
postgres_client_transaction::LogTransactionRequest,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
AccountsDbPluginError, ReplicaAccountInfo, SlotStatus,
AccountsDbPluginError, ReplicaAccountInfo, ReplicaBlockInfo, SlotStatus,
},
solana_measure::measure::Measure,
solana_metrics::*,
@ -44,6 +47,7 @@ struct PostgresSqlClientWrapper {
update_slot_with_parent_stmt: Statement,
update_slot_without_parent_stmt: Statement,
update_transaction_log_stmt: Statement,
update_block_metadata_stmt: Statement,
}
pub struct SimplePostgresClient {
@ -195,6 +199,11 @@ pub trait PostgresClient {
&mut self,
transaction_log_info: LogTransactionRequest,
) -> Result<(), AccountsDbPluginError>;
fn update_block_metadata(
&mut self,
block_info: UpdateBlockMetadataRequest,
) -> Result<(), AccountsDbPluginError>;
}
impl SimplePostgresClient {
@ -501,6 +510,8 @@ impl SimplePostgresClient {
Self::build_slot_upsert_statement_without_parent(&mut client, config)?;
let update_transaction_log_stmt =
Self::build_transaction_info_upsert_statement(&mut client, config)?;
let update_block_metadata_stmt =
Self::build_block_metadata_upsert_statement(&mut client, config)?;
let batch_size = config
.batch_size
@ -516,6 +527,7 @@ impl SimplePostgresClient {
update_slot_with_parent_stmt,
update_slot_without_parent_stmt,
update_transaction_log_stmt,
update_block_metadata_stmt,
}),
})
}
@ -591,6 +603,13 @@ impl PostgresClient for SimplePostgresClient {
) -> Result<(), AccountsDbPluginError> {
self.log_transaction_impl(transaction_log_info)
}
fn update_block_metadata(
&mut self,
block_info: UpdateBlockMetadataRequest,
) -> Result<(), AccountsDbPluginError> {
self.update_block_metadata_impl(block_info)
}
}
struct UpdateAccountRequest {
@ -604,11 +623,16 @@ struct UpdateSlotRequest {
slot_status: SlotStatus,
}
pub struct UpdateBlockMetadataRequest {
pub block_info: DbBlockInfo,
}
#[warn(clippy::large_enum_variant)]
enum DbWorkItem {
UpdateAccount(Box<UpdateAccountRequest>),
UpdateSlot(Box<UpdateSlotRequest>),
LogTransaction(Box<LogTransactionRequest>),
UpdateBlockMetadata(Box<UpdateBlockMetadataRequest>),
}
impl PostgresClientWorker {
@ -677,6 +701,14 @@ impl PostgresClientWorker {
}
}
}
DbWorkItem::UpdateBlockMetadata(block_info) => {
if let Err(err) = self.client.update_block_metadata(*block_info) {
error!("Failed to update block metadata: ({})", err);
if panic_on_db_errors {
abort();
}
}
}
},
Err(err) => match err {
RecvTimeoutError::Timeout => {
@ -868,6 +900,25 @@ impl ParallelPostgresClient {
Ok(())
}
pub fn update_block_metadata(
&mut self,
block_info: &ReplicaBlockInfo,
) -> Result<(), AccountsDbPluginError> {
if let Err(err) = self.sender.send(DbWorkItem::UpdateBlockMetadata(Box::new(
UpdateBlockMetadataRequest {
block_info: DbBlockInfo::from(block_info),
},
))) {
return Err(AccountsDbPluginError::SlotStatusUpdateError {
msg: format!(
"Failed to update the block metadata at slot {:?}, error: {:?}",
block_info.slot, err
),
});
}
Ok(())
}
pub fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError> {
info!("Notifying the end of startup");
// Ensure all items in the queue has been received by the workers

97
accountsdb-plugin-postgres/src/postgres_client/postgres_client_block_metadata.rs Normal file
View File

@ -0,0 +1,97 @@
use {
crate::{
accountsdb_plugin_postgres::{
AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
},
postgres_client::{
postgres_client_transaction::DbReward, SimplePostgresClient, UpdateBlockMetadataRequest,
},
},
chrono::Utc,
log::*,
postgres::{Client, Statement},
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
AccountsDbPluginError, ReplicaBlockInfo,
},
};
#[derive(Clone, Debug)]
pub struct DbBlockInfo {
pub slot: i64,
pub blockhash: String,
pub rewards: Vec<DbReward>,
pub block_time: Option<i64>,
pub block_height: Option<i64>,
}
impl<'a> From<&ReplicaBlockInfo<'a>> for DbBlockInfo {
fn from(block_info: &ReplicaBlockInfo) -> Self {
Self {
slot: block_info.slot as i64,
blockhash: block_info.blockhash.to_string(),
rewards: block_info.rewards.iter().map(DbReward::from).collect(),
block_time: block_info.block_time,
block_height: block_info
.block_height
.map(|block_height| block_height as i64),
}
}
}
impl SimplePostgresClient {
pub(crate) fn build_block_metadata_upsert_statement(
client: &mut Client,
config: &AccountsDbPluginPostgresConfig,
) -> Result<Statement, AccountsDbPluginError> {
let stmt =
"INSERT INTO block (slot, blockhash, rewards, block_time, block_height, updated_on) \
VALUES ($1, $2, $3, $4, $5, $6)";
let stmt = client.prepare(stmt);
match stmt {
Err(err) => {
return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
msg: format!(
"Error in preparing for the block metadata update PostgreSQL database: ({}) host: {:?} user: {:?} config: {:?}",
err, config.host, config.user, config
),
})));
}
Ok(stmt) => Ok(stmt),
}
}
pub(crate) fn update_block_metadata_impl(
&mut self,
block_info: UpdateBlockMetadataRequest,
) -> Result<(), AccountsDbPluginError> {
let client = self.client.get_mut().unwrap();
let statement = &client.update_block_metadata_stmt;
let client = &mut client.client;
let updated_on = Utc::now().naive_utc();
let block_info = block_info.block_info;
let result = client.query(
statement,
&[
&block_info.slot,
&block_info.blockhash,
&block_info.rewards,
&block_info.block_time,
&block_info.block_height,
&updated_on,
],
);
if let Err(err) = result {
let msg = format!(
"Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}",
err);
error!("{}", msg);
return Err(AccountsDbPluginError::AccountsUpdateError { msg });
}
Ok(())
}
}