gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

v1.9: Enforce tx metadata upload to bigtable (#23212)

Browse Source 
* Enforce tx metadata upload with static types (#23028)

* resolve conflicts

* fix test
This commit is contained in:
Justin Starry
2022-02-23 12:03:17 +08:00
committed by GitHub
parent 8413700a2f
commit 450404f800
16 changed files with 367 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
storage-bigtable/src/bigtable.rs
View File

@@ -793,19 +793,42 @@ mod tests {
solana_sdk::{hash::Hash, signature::Keypair, system_transaction},
solana_storage_proto::convert::generated,
solana_transaction_status::{
ConfirmedBlock, TransactionStatusMeta, TransactionWithStatusMeta,
ConfirmedBlock, ConfirmedBlockWithOptionalMetadata, TransactionStatusMeta,
TransactionWithMetadata,
},
std::convert::TryInto,
};
fn confirmed_block_into_protobuf(confirmed_block: ConfirmedBlock) -> generated::ConfirmedBlock {
let ConfirmedBlock {
previous_blockhash,
blockhash,
parent_slot,
transactions,
rewards,
block_time,
block_height,
} = confirmed_block;
generated::ConfirmedBlock {
previous_blockhash,
blockhash,
parent_slot,
transactions: transactions.into_iter().map(Into::into).collect(),
rewards: rewards.into_iter().map(|r| r.into()).collect(),
block_time: block_time.map(|timestamp| generated::UnixTimestamp { timestamp }),
block_height: block_height.map(|block_height| generated::BlockHeight { block_height }),
}
}
#[test]
fn test_deserialize_protobuf_or_bincode_cell_data() {
let from = Keypair::new();
let recipient = solana_sdk::pubkey::new_rand();
let transaction = system_transaction::transfer(&from, &recipient, 42, Hash::default());
let with_meta = TransactionWithStatusMeta {
let with_meta = TransactionWithMetadata {
transaction,
meta: Some(TransactionStatusMeta {
meta: TransactionStatusMeta {
status: Ok(()),
fee: 1,
pre_balances: vec![43, 0, 1],
@@ -815,9 +838,9 @@ mod tests {
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
rewards: Some(vec![]),
}),
},
};
let block = ConfirmedBlock {
let expected_block = ConfirmedBlock {
transactions: vec![with_meta],
parent_slot: 1,
blockhash: Hash::default().to_string(),
@@ -827,11 +850,14 @@ mod tests {
block_height: Some(1),
};
let bincode_block = compress_best(
&bincode::serialize::<StoredConfirmedBlock>(&block.clone().into()).unwrap(),
&bincode::serialize::<StoredConfirmedBlock>(
&ConfirmedBlockWithOptionalMetadata::from(expected_block.clone()).into(),
)
.unwrap(),
)
.unwrap();
let protobuf_block = generated::ConfirmedBlock::from(block.clone());
let protobuf_block = confirmed_block_into_protobuf(expected_block.clone());
let mut buf = Vec::with_capacity(protobuf_block.encoded_len());
protobuf_block.encode(&mut buf).unwrap();
let protobuf_block = compress_best(&buf).unwrap();
@@ -846,7 +872,10 @@ mod tests {
)
.unwrap();
if let CellData::Protobuf(protobuf_block) = deserialized {
assert_eq!(block, protobuf_block.try_into().unwrap());
assert_eq!(
ConfirmedBlockWithOptionalMetadata::from(expected_block.clone()),
protobuf_block.try_into().unwrap(),
);
} else {
panic!("deserialization should produce CellData::Protobuf");
}
@@ -861,15 +890,19 @@ mod tests {
)
.unwrap();
if let CellData::Bincode(bincode_block) = deserialized {
let mut block = block;
if let Some(meta) = &mut block.transactions[0].meta {
let mut block = expected_block;
{
let mut meta = &mut block.transactions[0].meta;
meta.inner_instructions = None; // Legacy bincode implementation does not support inner_instructions
meta.log_messages = None; // Legacy bincode implementation does not support log_messages
meta.pre_token_balances = None; // Legacy bincode implementation does not support token balances
meta.post_token_balances = None; // Legacy bincode implementation does not support token balances
meta.rewards = None; // Legacy bincode implementation does not support rewards
}
assert_eq!(block, bincode_block.into());
assert_eq!(
ConfirmedBlockWithOptionalMetadata::from(block),
ConfirmedBlockWithOptionalMetadata::from(bincode_block)
);
} else {
panic!("deserialization should produce CellData::Bincode");
}

39
storage-bigtable/src/lib.rs
View File

@@ -13,10 +13,10 @@ use {
},
solana_storage_proto::convert::{generated, tx_by_addr},
solana_transaction_status::{
extract_and_fmt_memos, ConfirmedBlock, ConfirmedTransaction,
ConfirmedTransactionStatusWithSignature, Reward, TransactionByAddrInfo,
TransactionConfirmationStatus, TransactionStatus, TransactionStatusMeta,
TransactionWithStatusMeta,
extract_and_fmt_memos, ConfirmedBlock, ConfirmedBlockWithOptionalMetadata,
ConfirmedTransactionStatusWithSignature, ConfirmedTransactionWithOptionalMetadata, Reward,
TransactionByAddrInfo, TransactionConfirmationStatus, TransactionStatus,
TransactionStatusMeta, TransactionWithMetadata, TransactionWithOptionalMetadata,
},
std::{
collections::{HashMap, HashSet},
@@ -114,9 +114,10 @@ struct StoredConfirmedBlock {
block_height: Option<u64>,
}
impl From<ConfirmedBlock> for StoredConfirmedBlock {
fn from(confirmed_block: ConfirmedBlock) -> Self {
let ConfirmedBlock {
#[cfg(test)]
impl From<ConfirmedBlockWithOptionalMetadata> for StoredConfirmedBlock {
fn from(confirmed_block: ConfirmedBlockWithOptionalMetadata) -> Self {
let ConfirmedBlockWithOptionalMetadata {
previous_blockhash,
blockhash,
parent_slot,
@@ -138,7 +139,7 @@ impl From<ConfirmedBlock> for StoredConfirmedBlock {
}
}
impl From<StoredConfirmedBlock> for ConfirmedBlock {
impl From<StoredConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
fn from(confirmed_block: StoredConfirmedBlock) -> Self {
let StoredConfirmedBlock {
previous_blockhash,
@@ -168,8 +169,9 @@ struct StoredConfirmedBlockTransaction {
meta: Option<StoredConfirmedBlockTransactionStatusMeta>,
}
impl From<TransactionWithStatusMeta> for StoredConfirmedBlockTransaction {
fn from(value: TransactionWithStatusMeta) -> Self {
#[cfg(test)]
impl From<TransactionWithOptionalMetadata> for StoredConfirmedBlockTransaction {
fn from(value: TransactionWithOptionalMetadata) -> Self {
Self {
transaction: value.transaction,
meta: value.meta.map(|meta| meta.into()),
@@ -177,7 +179,7 @@ impl From<TransactionWithStatusMeta> for StoredConfirmedBlockTransaction {
}
}
impl From<StoredConfirmedBlockTransaction> for TransactionWithStatusMeta {
impl From<StoredConfirmedBlockTransaction> for TransactionWithOptionalMetadata {
fn from(value: StoredConfirmedBlockTransaction) -> Self {
Self {
transaction: value.transaction,
@@ -392,7 +394,10 @@ impl LedgerStorage {
}
/// Fetch the confirmed block from the desired slot
pub async fn get_confirmed_block(&self, slot: Slot) -> Result<ConfirmedBlock> {
pub async fn get_confirmed_block(
&self,
slot: Slot,
) -> Result<ConfirmedBlockWithOptionalMetadata> {
debug!(
"LedgerStorage::get_confirmed_block request received: {:?}",
slot
@@ -438,7 +443,7 @@ impl LedgerStorage {
pub async fn get_confirmed_transaction(
&self,
signature: &Signature,
) -> Result<Option<ConfirmedTransaction>> {
) -> Result<Option<ConfirmedTransactionWithOptionalMetadata>> {
debug!(
"LedgerStorage::get_confirmed_transaction request received: {:?}",
signature
@@ -471,7 +476,7 @@ impl LedgerStorage {
);
Ok(None)
} else {
Ok(Some(ConfirmedTransaction {
Ok(Some(ConfirmedTransactionWithOptionalMetadata {
slot,
transaction: bucket_block_transaction,
block_time: block.block_time,
@@ -635,8 +640,8 @@ impl LedgerStorage {
let mut tx_cells = vec![];
for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() {
let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta;
let err = meta.as_ref().and_then(|meta| meta.status.clone().err());
let TransactionWithMetadata { meta, transaction } = transaction_with_meta;
let err = meta.status.clone().err();
let index = index as u32;
let signature = transaction.signatures[0];
let memo = extract_and_fmt_memos(&transaction.message);
@@ -723,7 +728,7 @@ impl LedgerStorage {
let mut expected_tx_infos: HashMap<String, UploadedTransaction> = HashMap::new();
let confirmed_block = self.get_confirmed_block(slot).await?;
for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() {
let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta;
let TransactionWithOptionalMetadata { meta, transaction } = transaction_with_meta;
let signature = transaction.signatures[0];
let index = index as u32;
let err = meta.as_ref().and_then(|meta| meta.status.clone().err());