gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

v1.8: Enforce tx metadata upload to bigtable (#23105)

Browse Source 
* v1.8: Enforce tx metadata upload to bigtable

* fix bpf ci step
This commit is contained in:
Justin Starry
2022-02-16 18:02:03 +08:00
committed by GitHub
parent 52c959e7f0
commit 3424ace92c
15 changed files with 243 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
core/src/banking_stage.rs
View File

@ -1925,7 +1925,7 @@ mod tests {
transaction::TransactionError,
},
solana_streamer::{recvmmsg::recv_mmsg, socket::SocketAddrSpace},
solana_transaction_status::TransactionWithStatusMeta,
solana_transaction_status::TransactionWithMetadata,
solana_vote_program::vote_transaction,
std::{
net::SocketAddr,
@ -3080,20 +3080,17 @@ mod tests {
let keypair1 = Keypair::new();
let success_tx =
system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash());
system_transaction::transfer(&mint_keypair, &pubkey, 0, genesis_config.hash());
let success_signature = success_tx.signatures[0];
let entry_1 = next_entry(&genesis_config.hash(), 1, vec![success_tx.clone()]);
let ix_error_tx =
system_transaction::transfer(&keypair1, &pubkey1, 10, genesis_config.hash());
system_transaction::transfer(&keypair1, &pubkey1, std::u64::MAX, genesis_config.hash());
let ix_error_signature = ix_error_tx.signatures[0];
let entry_2 = next_entry(&entry_1.hash, 1, vec![ix_error_tx.clone()]);
let fail_tx =
system_transaction::transfer(&mint_keypair, &pubkey1, 1, genesis_config.hash());
let entry_3 = next_entry(&entry_2.hash, 1, vec![fail_tx.clone()]);
let entries = vec![entry_1, entry_2, entry_3];
let entries = vec![entry_1, entry_2];
let transactions = vec![success_tx.into(), ix_error_tx.into(), fail_tx.into()];
bank.transfer(4, &mint_keypair, &keypair1.pubkey()).unwrap();
let transactions = vec![success_tx.into(), ix_error_tx.into()];
bank.transfer(1, &mint_keypair, &keypair1.pubkey()).unwrap();
let start = Arc::new(Instant::now());
let working_bank = WorkingBank {
@ -3155,27 +3152,24 @@ mod tests {
transaction_status_service.join().unwrap();
let confirmed_block = blockstore.get_rooted_block(bank.slot(), false).unwrap();
assert_eq!(confirmed_block.transactions.len(), 3);
for TransactionWithStatusMeta { transaction, meta } in
confirmed_block.transactions.into_iter()
{
if transaction.signatures[0] == success_signature {
let meta = meta.unwrap();
assert_eq!(meta.status, Ok(()));
} else if transaction.signatures[0] == ix_error_signature {
let meta = meta.unwrap();
assert_eq!(
meta.status,
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1)
))
);
} else {
assert_eq!(meta, None);
}
}
let actual_tx_results: Vec<_> = confirmed_block
.transactions
.into_iter()
.map(|TransactionWithMetadata { transaction, meta }| {
(transaction.signatures[0], meta.status)
})
.collect();
let expected_tx_results = vec![
(success_signature, Ok(())),
(
ix_error_signature,
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1),
)),
),
];
assert_eq!(actual_tx_results, expected_tx_results);
poh_recorder
.lock()

47
core/src/replay_stage.rs
View File

@ -2670,7 +2670,7 @@ mod tests {
transaction::TransactionError,
},
solana_streamer::socket::SocketAddrSpace,
solana_transaction_status::TransactionWithStatusMeta,
solana_transaction_status::TransactionWithMetadata,
solana_vote_program::{
vote_state::{VoteState, VoteStateVersions},
vote_transaction,
@ -3472,36 +3472,37 @@ mod tests {
let bank1 = Arc::new(Bank::new_from_parent(&bank0, &Pubkey::default(), 1));
let slot = bank1.slot();
let signatures = create_test_transactions_and_populate_blockstore(
let mut test_signatures_iter = create_test_transactions_and_populate_blockstore(
vec![&mint_keypair, &keypair1, &keypair2, &keypair3],
bank0.slot(),
bank1,
blockstore.clone(),
Arc::new(AtomicU64::default()),
);
)
.into_iter();
let confirmed_block = blockstore.get_rooted_block(slot, false).unwrap();
assert_eq!(confirmed_block.transactions.len(), 3);
let actual_tx_results: Vec<_> = confirmed_block
.transactions
.into_iter()
.map(|TransactionWithMetadata { transaction, meta }| {
(transaction.signatures[0], meta.status)
})
.collect();
for TransactionWithStatusMeta { transaction, meta } in
confirmed_block.transactions.into_iter()
{
if transaction.signatures[0] == signatures[0] {
let meta = meta.unwrap();
assert_eq!(meta.status, Ok(()));
} else if transaction.signatures[0] == signatures[1] {
let meta = meta.unwrap();
assert_eq!(
meta.status,
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1)
))
);
} else {
assert_eq!(meta, None);
}
}
let expected_tx_results = vec![
(test_signatures_iter.next().unwrap(), Ok(())),
(
test_signatures_iter.next().unwrap(),
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1),
)),
),
];
assert_eq!(actual_tx_results, expected_tx_results);
assert!(test_signatures_iter.next().is_none());
}
Blockstore::destroy(&ledger_path).unwrap();
}

16
ledger-tool/src/bigtable.rs
View File

@ -16,7 +16,9 @@ use {
},
solana_ledger::{blockstore::Blockstore, blockstore_db::AccessType},
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
solana_transaction_status::{ConfirmedBlock, EncodedTransaction, UiTransactionEncoding},
solana_transaction_status::{
ConfirmedBlockWithOptionalMetadata, EncodedTransaction, UiTransactionEncoding,
},
std::{
collections::HashSet,
path::Path,
@ -30,7 +32,6 @@ async fn upload(
blockstore: Blockstore,
starting_slot: Slot,
ending_slot: Option<Slot>,
allow_missing_metadata: bool,
force_reupload: bool,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
@ -42,7 +43,6 @@ async fn upload(
bigtable,
starting_slot,
ending_slot,
allow_missing_metadata,
force_reupload,
Arc::new(AtomicBool::new(false)),
)
@ -194,7 +194,7 @@ pub async fn transaction_history(
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(true, None, None).await?;
let mut loaded_block: Option<(Slot, ConfirmedBlock)> = None;
let mut loaded_block: Option<(Slot, ConfirmedBlockWithOptionalMetadata)> = None;
while limit > 0 {
let results = bigtable
.get_confirmed_signatures_for_address(
@ -306,12 +306,6 @@ impl BigTableSubCommand for App<'_, '_> {
.index(2)
.help("Stop uploading at this slot [default: last available slot]"),
)
.arg(
Arg::with_name("allow_missing_metadata")
.long("allow-missing-metadata")
.takes_value(false)
.help("Don't panic if transaction metadata is missing"),
)
.arg(
Arg::with_name("force_reupload")
.long("force")
@ -506,7 +500,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
("upload", Some(arg_matches)) => {
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
let force_reupload = arg_matches.is_present("force_reupload");
let blockstore = crate::open_blockstore(
&canonicalize_ledger_path(ledger_path),
@ -518,7 +511,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
blockstore,
starting_slot,
ending_slot,
allow_missing_metadata,
force_reupload,
))
}

16
ledger/src/bigtable_upload.rs
View File

@ -25,7 +25,6 @@ pub async fn upload_confirmed_blocks(
bigtable: solana_storage_bigtable::LedgerStorage,
starting_slot: Slot,
ending_slot: Option<Slot>,
allow_missing_metadata: bool,
force_reupload: bool,
exit: Arc<AtomicBool>,
) -> Result<(), Box<dyn std::error::Error>> {
@ -186,20 +185,7 @@ pub async fn upload_confirmed_blocks(
num_blocks -= 1;
None
}
Some(confirmed_block) => {
if confirmed_block
.transactions
.iter()
.any(|transaction| transaction.meta.is_none())
{
if allow_missing_metadata {
info!("Transaction metadata missing from slot {}", slot);
} else {
panic!("Transaction metadata missing from slot {}", slot);
}
}
Some(bigtable.upload_confirmed_block(slot, confirmed_block))
}
Some(confirmed_block) => Some(bigtable.upload_confirmed_block(slot, confirmed_block)),
});
for result in futures::future::join_all(uploads).await {

1
ledger/src/bigtable_upload_service.rs
View File

@ -72,7 +72,6 @@ impl BigTableUploadService {
bigtable_ledger_storage.clone(),
start_slot,
Some(end_slot),
true,
false,
exit.clone(),
));

56
ledger/src/blockstore.rs
View File

@ -42,7 +42,7 @@ use {
solana_storage_proto::{StoredExtendedRewards, StoredTransactionStatusMeta},
solana_transaction_status::{
ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards,
TransactionStatusMeta, TransactionWithStatusMeta,
TransactionStatusMeta, TransactionWithMetadata, TransactionWithOptionalMetadata,
},
std::{
borrow::Cow,
@ -1950,7 +1950,7 @@ impl Blockstore {
// from shreds received.
parent_slot: slot_meta.parent_slot.unwrap(),
transactions: self
.map_transactions_to_statuses(slot, slot_transaction_iterator),
.map_transactions_to_statuses(slot, slot_transaction_iterator)?,
rewards,
block_time,
block_height,
@ -1965,17 +1965,16 @@ impl Blockstore {
&self,
slot: Slot,
iterator: impl Iterator<Item = Transaction> + 'a,
) -> Vec<TransactionWithStatusMeta> {
) -> Result<Vec<TransactionWithMetadata>> {
iterator
.map(|transaction| {
let signature = transaction.signatures[0];
TransactionWithStatusMeta {
Ok(TransactionWithMetadata {
transaction,
meta: self
.read_transaction_status((signature, slot))
.ok()
.flatten(),
}
.read_transaction_status((signature, slot))?
.ok_or(BlockstoreError::MissingTransactionMetadata)?,
})
})
.collect()
}
@ -2256,7 +2255,7 @@ impl Blockstore {
let block_time = self.get_block_time(slot)?;
Ok(Some(ConfirmedTransaction {
slot,
transaction: TransactionWithStatusMeta {
transaction: TransactionWithOptionalMetadata {
transaction,
meta: Some(status),
},
@ -6082,7 +6081,7 @@ pub mod tests {
.put_meta_bytes(slot - 1, &serialize(&parent_meta).unwrap())
.unwrap();
let expected_transactions: Vec<TransactionWithStatusMeta> = entries
let expected_transactions: Vec<TransactionWithMetadata> = entries
.iter()
.cloned()
.filter(|entry| !entry.is_tick())
@ -6143,9 +6142,9 @@ pub mod tests {
.transaction_status_cf
.put_protobuf((0, signature, slot + 2), &status)
.unwrap();
TransactionWithStatusMeta {
TransactionWithMetadata {
transaction,
meta: Some(TransactionStatusMeta {
meta: TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances,
@ -6155,7 +6154,7 @@ pub mod tests {
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
rewards: Some(vec![]),
}),
},
}
})
.collect();
@ -6952,7 +6951,7 @@ pub mod tests {
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(vec![slot - 1, slot].iter()).unwrap();
let expected_transactions: Vec<TransactionWithStatusMeta> = entries
let expected_transactions: Vec<TransactionWithOptionalMetadata> = entries
.iter()
.cloned()
.filter(|entry| !entry.is_tick())
@ -6989,7 +6988,7 @@ pub mod tests {
.transaction_status_cf
.put_protobuf((0, signature, slot), &status)
.unwrap();
TransactionWithStatusMeta {
TransactionWithOptionalMetadata {
transaction,
meta: Some(TransactionStatusMeta {
status: Ok(()),
@ -7030,7 +7029,7 @@ pub mod tests {
blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap();
*blockstore.lowest_cleanup_slot.write().unwrap() = slot;
for TransactionWithStatusMeta { transaction, .. } in expected_transactions {
for TransactionWithOptionalMetadata { transaction, .. } in expected_transactions {
let signature = transaction.signatures[0];
assert_eq!(blockstore.get_rooted_transaction(signature).unwrap(), None,);
assert_eq!(
@ -7051,7 +7050,7 @@ pub mod tests {
let blockstore = Blockstore::open(&ledger_path).unwrap();
blockstore.insert_shreds(shreds, None, false).unwrap();
let expected_transactions: Vec<TransactionWithStatusMeta> = entries
let expected_transactions: Vec<TransactionWithOptionalMetadata> = entries
.iter()
.cloned()
.filter(|entry| !entry.is_tick())
@ -7088,7 +7087,7 @@ pub mod tests {
.transaction_status_cf
.put_protobuf((0, signature, slot), &status)
.unwrap();
TransactionWithStatusMeta {
TransactionWithOptionalMetadata {
transaction,
meta: Some(TransactionStatusMeta {
status: Ok(()),
@ -7122,7 +7121,7 @@ pub mod tests {
blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap();
*blockstore.lowest_cleanup_slot.write().unwrap() = slot;
for TransactionWithStatusMeta { transaction, .. } in expected_transactions {
for TransactionWithOptionalMetadata { transaction, .. } in expected_transactions {
let signature = transaction.signatures[0];
assert_eq!(
blockstore
@ -7877,6 +7876,16 @@ pub mod tests {
.unwrap();
transactions.push(transaction);
}
let map_result =
blockstore.map_transactions_to_statuses(slot, transactions.clone().into_iter());
assert!(map_result.is_ok());
let map = map_result.unwrap();
assert_eq!(map.len(), 4);
for (x, m) in map.iter().enumerate() {
assert_eq!(m.meta.fee, x as u64);
}
// Push transaction that will not have matching status, as a test case
transactions.push(Transaction::new_with_compiled_instructions(
&[&Keypair::new()],
@ -7886,12 +7895,9 @@ pub mod tests {
vec![CompiledInstruction::new(1, &(), vec![0])],
));
let map = blockstore.map_transactions_to_statuses(slot, transactions.into_iter());
assert_eq!(map.len(), 5);
for (x, m) in map.iter().take(4).enumerate() {
assert_eq!(m.meta.as_ref().unwrap().fee, x as u64);
}
assert_eq!(map[4].meta, None);
let map_result =
blockstore.map_transactions_to_statuses(slot, transactions.into_iter());
assert_matches!(map_result, Err(BlockstoreError::MissingTransactionMetadata));
}
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}

1
ledger/src/blockstore_db.rs
View File

@ -101,6 +101,7 @@ pub enum BlockstoreError {
ProtobufDecodeError(#[from] prost::DecodeError),
ParentEntriesUnavailable,
SlotUnavailable,
MissingTransactionMetadata,
}
pub type Result<T> = std::result::Result<T, BlockstoreError>;

4
local-cluster/tests/common.rs
View File

@ -250,6 +250,10 @@ pub fn run_cluster_partition<C>(
..ValidatorConfig::default_for_test()
};
// Enabled so that transaction statuses are written to blockstore which enables
// tests to detect when a block has been replayed.
validator_config.rpc_config.enable_rpc_transaction_history = true;
// Returns:
// 1) The keys for the validators
// 2) The amount of time it would take to iterate through one full iteration of the given

12
local-cluster/tests/local_cluster_slow.rs
View File

@ -725,13 +725,19 @@ fn find_latest_replayed_slot_from_ledger(
// Wait for the slot to be replayed
loop {
info!("Waiting for slot {} to be replayed", latest_slot);
if !blockstore
let replayed_transactions = blockstore
.map_transactions_to_statuses(
latest_slot,
non_tick_entry.transactions.clone().into_iter(),
)
.is_empty()
{
.unwrap_or_else(|_| {
info!(
"Transaction statuses for slot {} haven't been written yet",
latest_slot
);
Vec::new()
});
if !replayed_transactions.is_empty() {
return (
latest_slot,
AncestorIterator::new(latest_slot, &blockstore).collect(),

4
programs/bpf/tests/programs.rs
View File

@ -50,7 +50,7 @@ use solana_sdk::{
};
use solana_transaction_status::{
token_balances::collect_token_balances, ConfirmedTransaction, InnerInstructions,
TransactionStatusMeta, TransactionWithStatusMeta, UiTransactionEncoding,
TransactionStatusMeta, TransactionWithOptionalMetadata, UiTransactionEncoding,
};
use std::{
cell::RefCell, collections::HashMap, env, fs::File, io::Read, path::PathBuf, str::FromStr,
@ -403,7 +403,7 @@ fn execute_transactions(bank: &Bank, txs: &[Transaction]) -> Vec<ConfirmedTransa
ConfirmedTransaction {
slot: bank.slot(),
transaction: TransactionWithStatusMeta {
transaction: TransactionWithOptionalMetadata {
transaction: tx.clone(),
meta: Some(tx_status_meta),
},

56
rpc/src/rpc.rs
View File

@ -73,9 +73,9 @@ use {
solana_storage_bigtable::Error as StorageError,
solana_streamer::socket::SocketAddrSpace,
solana_transaction_status::{
ConfirmedBlock, ConfirmedTransactionStatusWithSignature, EncodedConfirmedTransaction,
Reward, RewardType, TransactionConfirmationStatus, TransactionStatus, UiConfirmedBlock,
UiTransactionEncoding,
ConfirmedBlockWithOptionalMetadata, ConfirmedTransactionStatusWithSignature,
EncodedConfirmedTransaction, Reward, RewardType, TransactionConfirmationStatus,
TransactionStatus, UiConfirmedBlock, UiTransactionEncoding,
},
solana_vote_program::vote_state::{VoteState, MAX_LOCKOUT_HISTORY},
spl_token::{
@ -990,7 +990,7 @@ impl JsonRpcRequestProcessor {
self.check_status_is_complete(slot)?;
let result = self.blockstore.get_rooted_block(slot, true);
self.check_blockstore_root(&result, slot)?;
let configure_block = |confirmed_block: ConfirmedBlock| {
let configure_block = |confirmed_block: ConfirmedBlockWithOptionalMetadata| {
let mut confirmed_block =
confirmed_block.configure(encoding, transaction_details, show_rewards);
if slot == 0 {
@ -1008,7 +1008,10 @@ impl JsonRpcRequestProcessor {
}
}
self.check_slot_cleaned_up(&result, slot)?;
return Ok(result.ok().map(configure_block));
return Ok(result
.ok()
.map(ConfirmedBlockWithOptionalMetadata::from)
.map(configure_block));
} else if commitment.is_confirmed() {
// Check if block is confirmed
let confirmed_bank = self.bank(Some(CommitmentConfig::confirmed()));
@ -1030,7 +1033,11 @@ impl JsonRpcRequestProcessor {
}
}
}
confirmed_block.configure(encoding, transaction_details, show_rewards)
ConfirmedBlockWithOptionalMetadata::from(confirmed_block).configure(
encoding,
transaction_details,
show_rewards,
)
}));
}
}
@ -4056,15 +4063,7 @@ pub fn create_test_transactions_and_populate_blockstore(
solana_sdk::system_transaction::transfer(keypair2, &keypair3.pubkey(), 10, blockhash);
let ix_error_signature = ix_error_tx.signatures[0];
let entry_2 = solana_ledger::entry::next_entry(&entry_1.hash, 1, vec![ix_error_tx]);
// Failed transaction
let fail_tx = solana_sdk::system_transaction::transfer(
mint_keypair,
&keypair2.pubkey(),
2,
Hash::default(),
);
let entry_3 = solana_ledger::entry::next_entry(&entry_2.hash, 1, vec![fail_tx]);
let mut entries = vec![entry_1, entry_2, entry_3];
let mut entries = vec![entry_1, entry_2];
let shreds = solana_ledger::blockstore::entries_to_test_shreds(
entries.clone(),
@ -4088,17 +4087,20 @@ pub fn create_test_transactions_and_populate_blockstore(
// Check that process_entries successfully writes can_commit transactions statuses, and
// that they are matched properly by get_rooted_block
let _result = solana_ledger::blockstore_processor::process_entries(
&bank,
&mut entries,
true,
Some(
&solana_ledger::blockstore_processor::TransactionStatusSender {
sender: transaction_status_sender,
enable_cpi_and_log_storage: false,
},
assert_eq!(
solana_ledger::blockstore_processor::process_entries(
&bank,
&mut entries,
true,
Some(
&solana_ledger::blockstore_processor::TransactionStatusSender {
sender: transaction_status_sender,
enable_cpi_and_log_storage: false,
},
),
Some(&replay_vote_sender),
),
Some(&replay_vote_sender),
Ok(())
);
transaction_status_service.join().unwrap();
@ -6264,7 +6266,7 @@ pub mod tests {
let confirmed_block: Option<EncodedConfirmedBlock> =
serde_json::from_value(result["result"].clone()).unwrap();
let confirmed_block = confirmed_block.unwrap();
assert_eq!(confirmed_block.transactions.len(), 3);
assert_eq!(confirmed_block.transactions.len(), 2);
assert_eq!(confirmed_block.rewards, vec![]);
for EncodedTransactionWithStatusMeta { transaction, meta } in
@ -6309,7 +6311,7 @@ pub mod tests {
let confirmed_block: Option<EncodedConfirmedBlock> =
serde_json::from_value(result["result"].clone()).unwrap();
let confirmed_block = confirmed_block.unwrap();
assert_eq!(confirmed_block.transactions.len(), 3);
assert_eq!(confirmed_block.transactions.len(), 2);
assert_eq!(confirmed_block.rewards, vec![]);
for EncodedTransactionWithStatusMeta { transaction, meta } in

33
storage-bigtable/src/bigtable.rs
View File

@ -793,17 +793,42 @@ mod tests {
solana_sdk::{hash::Hash, signature::Keypair, system_transaction},
solana_storage_proto::convert::generated,
solana_transaction_status::{
ConfirmedBlock, TransactionStatusMeta, TransactionWithStatusMeta,
ConfirmedBlockWithOptionalMetadata, TransactionStatusMeta,
TransactionWithOptionalMetadata,
},
std::convert::TryInto,
};
fn convert_from_block_with_optional_meta(
confirmed_block: ConfirmedBlockWithOptionalMetadata,
) -> generated::ConfirmedBlock {
let ConfirmedBlockWithOptionalMetadata {
previous_blockhash,
blockhash,
parent_slot,
transactions,
rewards,
block_time,
block_height,
} = confirmed_block;
generated::ConfirmedBlock {
previous_blockhash,
blockhash,
parent_slot,
transactions: transactions.into_iter().map(|tx| tx.into()).collect(),
rewards: rewards.into_iter().map(|r| r.into()).collect(),
block_time: block_time.map(|timestamp| generated::UnixTimestamp { timestamp }),
block_height: block_height.map(|block_height| generated::BlockHeight { block_height }),
}
}
#[test]
fn test_deserialize_protobuf_or_bincode_cell_data() {
let from = Keypair::new();
let recipient = solana_sdk::pubkey::new_rand();
let transaction = system_transaction::transfer(&from, &recipient, 42, Hash::default());
let with_meta = TransactionWithStatusMeta {
let with_meta = TransactionWithOptionalMetadata {
transaction,
meta: Some(TransactionStatusMeta {
status: Ok(()),
@ -817,7 +842,7 @@ mod tests {
rewards: Some(vec![]),
}),
};
let block = ConfirmedBlock {
let block = ConfirmedBlockWithOptionalMetadata {
transactions: vec![with_meta],
parent_slot: 1,
blockhash: Hash::default().to_string(),
@ -831,7 +856,7 @@ mod tests {
)
.unwrap();
let protobuf_block = generated::ConfirmedBlock::from(block.clone());
let protobuf_block = convert_from_block_with_optional_meta(block.clone());
let mut buf = Vec::with_capacity(protobuf_block.encoded_len());
protobuf_block.encode(&mut buf).unwrap();
let protobuf_block = compress_best(&buf).unwrap();

33
storage-bigtable/src/lib.rs
View File

@ -13,10 +13,10 @@ use {
},
solana_storage_proto::convert::{generated, tx_by_addr},
solana_transaction_status::{
extract_and_fmt_memos, ConfirmedBlock, ConfirmedTransaction,
ConfirmedTransactionStatusWithSignature, Reward, TransactionByAddrInfo,
TransactionConfirmationStatus, TransactionStatus, TransactionStatusMeta,
TransactionWithStatusMeta,
extract_and_fmt_memos, ConfirmedBlock, ConfirmedBlockWithOptionalMetadata,
ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Reward,
TransactionByAddrInfo, TransactionConfirmationStatus, TransactionStatus,
TransactionStatusMeta, TransactionWithMetadata, TransactionWithOptionalMetadata,
},
std::{
collections::{HashMap, HashSet},
@ -114,9 +114,9 @@ struct StoredConfirmedBlock {
block_height: Option<u64>,
}
impl From<ConfirmedBlock> for StoredConfirmedBlock {
fn from(confirmed_block: ConfirmedBlock) -> Self {
let ConfirmedBlock {
impl From<ConfirmedBlockWithOptionalMetadata> for StoredConfirmedBlock {
fn from(confirmed_block: ConfirmedBlockWithOptionalMetadata) -> Self {
let ConfirmedBlockWithOptionalMetadata {
previous_blockhash,
blockhash,
parent_slot,
@ -138,7 +138,7 @@ impl From<ConfirmedBlock> for StoredConfirmedBlock {
}
}
impl From<StoredConfirmedBlock> for ConfirmedBlock {
impl From<StoredConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
fn from(confirmed_block: StoredConfirmedBlock) -> Self {
let StoredConfirmedBlock {
previous_blockhash,
@ -168,8 +168,8 @@ struct StoredConfirmedBlockTransaction {
meta: Option<StoredConfirmedBlockTransactionStatusMeta>,
}
impl From<TransactionWithStatusMeta> for StoredConfirmedBlockTransaction {
fn from(value: TransactionWithStatusMeta) -> Self {
impl From<TransactionWithOptionalMetadata> for StoredConfirmedBlockTransaction {
fn from(value: TransactionWithOptionalMetadata) -> Self {
Self {
transaction: value.transaction,
meta: value.meta.map(|meta| meta.into()),
@ -177,7 +177,7 @@ impl From<TransactionWithStatusMeta> for StoredConfirmedBlockTransaction {
}
}
impl From<StoredConfirmedBlockTransaction> for TransactionWithStatusMeta {
impl From<StoredConfirmedBlockTransaction> for TransactionWithOptionalMetadata {
fn from(value: StoredConfirmedBlockTransaction) -> Self {
Self {
transaction: value.transaction,
@ -392,7 +392,10 @@ impl LedgerStorage {
}
/// Fetch the confirmed block from the desired slot
pub async fn get_confirmed_block(&self, slot: Slot) -> Result<ConfirmedBlock> {
pub async fn get_confirmed_block(
&self,
slot: Slot,
) -> Result<ConfirmedBlockWithOptionalMetadata> {
debug!(
"LedgerStorage::get_confirmed_block request received: {:?}",
slot
@ -635,8 +638,8 @@ impl LedgerStorage {
let mut tx_cells = vec![];
for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() {
let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta;
let err = meta.as_ref().and_then(|meta| meta.status.clone().err());
let TransactionWithMetadata { meta, transaction } = transaction_with_meta;
let err = meta.status.clone().err();
let index = index as u32;
let signature = transaction.signatures[0];
let memo = extract_and_fmt_memos(&transaction.message);
@ -723,7 +726,7 @@ impl LedgerStorage {
let mut expected_tx_infos: HashMap<String, UploadedTransaction> = HashMap::new();
let confirmed_block = self.get_confirmed_block(slot).await?;
for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() {
let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta;
let TransactionWithOptionalMetadata { meta, transaction } = transaction_with_meta;
let signature = transaction.signatures[0];
let index = index as u32;
let err = meta.as_ref().and_then(|meta| meta.status.clone().err());

28
storage-proto/src/convert.rs
View File

@ -10,8 +10,9 @@ use {
transaction::{Transaction, TransactionError},
},
solana_transaction_status::{
ConfirmedBlock, InnerInstructions, Reward, RewardType, TransactionByAddrInfo,
TransactionStatusMeta, TransactionTokenBalance, TransactionWithStatusMeta,
ConfirmedBlock, ConfirmedBlockWithOptionalMetadata, InnerInstructions, Reward, RewardType,
TransactionByAddrInfo, TransactionStatusMeta, TransactionTokenBalance,
TransactionWithMetadata, TransactionWithOptionalMetadata,
},
std::{
convert::{TryFrom, TryInto},
@ -135,7 +136,7 @@ impl From<ConfirmedBlock> for generated::ConfirmedBlock {
}
}
impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlock {
impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
type Error = bincode::Error;
fn try_from(
confirmed_block: generated::ConfirmedBlock,
@ -157,7 +158,8 @@ impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlock {
transactions: transactions
.into_iter()
.map(|tx| tx.try_into())
.collect::<std::result::Result<Vec<TransactionWithStatusMeta>, Self::Error>>()?,
.collect::<std::result::Result<Vec<TransactionWithOptionalMetadata>, Self::Error>>(
)?,
rewards: rewards.into_iter().map(|r| r.into()).collect(),
block_time: block_time.map(|generated::UnixTimestamp { timestamp }| timestamp),
block_height: block_height.map(|generated::BlockHeight { block_height }| block_height),
@ -165,17 +167,25 @@ impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlock {
}
}
impl From<TransactionWithStatusMeta> for generated::ConfirmedTransaction {
fn from(value: TransactionWithStatusMeta) -> Self {
let meta = value.meta.map(|meta| meta.into());
impl From<TransactionWithOptionalMetadata> for generated::ConfirmedTransaction {
fn from(value: TransactionWithOptionalMetadata) -> Self {
Self {
transaction: Some(value.transaction.into()),
meta,
meta: value.meta.map(|meta| meta.into()),
}
}
}
impl TryFrom<generated::ConfirmedTransaction> for TransactionWithStatusMeta {
impl From<TransactionWithMetadata> for generated::ConfirmedTransaction {
fn from(value: TransactionWithMetadata) -> Self {
Self {
transaction: Some(value.transaction.into()),
meta: Some(value.meta.into()),
}
}
}
impl TryFrom<generated::ConfirmedTransaction> for TransactionWithOptionalMetadata {
type Error = bincode::Error;
fn try_from(value: generated::ConfirmedTransaction) -> std::result::Result<Self, Self::Error> {
let meta = value.meta.map(|meta| meta.try_into()).transpose()?;

60
transaction-status/src/lib.rs
View File

@ -355,19 +355,52 @@ pub struct Reward {
pub type Rewards = Vec<Reward>;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
#[derive(Clone, Debug, PartialEq)]
pub struct ConfirmedBlock {
pub previous_blockhash: String,
pub blockhash: String,
pub parent_slot: Slot,
pub transactions: Vec<TransactionWithStatusMeta>,
pub transactions: Vec<TransactionWithMetadata>,
pub rewards: Rewards,
pub block_time: Option<UnixTimestamp>,
pub block_height: Option<u64>,
}
impl ConfirmedBlock {
#[derive(Clone, Debug, PartialEq)]
pub struct ConfirmedBlockWithOptionalMetadata {
pub previous_blockhash: String,
pub blockhash: String,
pub parent_slot: Slot,
pub transactions: Vec<TransactionWithOptionalMetadata>,
pub rewards: Rewards,
pub block_time: Option<UnixTimestamp>,
pub block_height: Option<u64>,
}
impl From<ConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
fn from(block: ConfirmedBlock) -> Self {
Self {
previous_blockhash: block.previous_blockhash,
blockhash: block.blockhash,
parent_slot: block.parent_slot,
transactions: block
.transactions
.into_iter()
.map(|TransactionWithMetadata { transaction, meta }| {
TransactionWithOptionalMetadata {
transaction,
meta: Some(meta),
}
})
.collect(),
rewards: block.rewards,
block_time: block.block_time,
block_height: block.block_height,
}
}
}
impl ConfirmedBlockWithOptionalMetadata {
pub fn encode(self, encoding: UiTransactionEncoding) -> EncodedConfirmedBlock {
EncodedConfirmedBlock {
previous_blockhash: self.previous_blockhash,
@ -499,12 +532,10 @@ impl Default for TransactionDetails {
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
#[derive(Debug, Clone, PartialEq)]
pub struct ConfirmedTransaction {
pub slot: Slot,
#[serde(flatten)]
pub transaction: TransactionWithStatusMeta,
pub transaction: TransactionWithOptionalMetadata,
pub block_time: Option<UnixTimestamp>,
}
@ -561,14 +592,19 @@ pub struct UiParsedMessage {
pub instructions: Vec<UiInstruction>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TransactionWithStatusMeta {
#[derive(Clone, Debug, PartialEq)]
pub struct TransactionWithOptionalMetadata {
pub transaction: Transaction,
pub meta: Option<TransactionStatusMeta>,
}
impl TransactionWithStatusMeta {
#[derive(Clone, Debug, PartialEq)]
pub struct TransactionWithMetadata {
pub transaction: Transaction,
pub meta: TransactionStatusMeta,
}
impl TransactionWithOptionalMetadata {
fn encode(self, encoding: UiTransactionEncoding) -> EncodedTransactionWithStatusMeta {
let message = self.transaction.message();
let meta = self.meta.map(|meta| meta.encode(encoding, message));