gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

v1.8: Enforce tx metadata upload to bigtable (#23105)

Browse Source 
* v1.8: Enforce tx metadata upload to bigtable

* fix bpf ci step
This commit is contained in:
Justin Starry
2022-02-16 18:02:03 +08:00
committed by GitHub
parent 52c959e7f0
commit 3424ace92c
15 changed files with 243 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
core/src/banking_stage.rs
View File

@ -1925,7 +1925,7 @@ mod tests {
transaction::TransactionError, transaction::TransactionError,
}, },
solana_streamer::{recvmmsg::recv_mmsg, socket::SocketAddrSpace}, solana_streamer::{recvmmsg::recv_mmsg, socket::SocketAddrSpace},
solana_transaction_status::TransactionWithStatusMeta, solana_transaction_status::TransactionWithMetadata,
solana_vote_program::vote_transaction, solana_vote_program::vote_transaction,
std::{ std::{
net::SocketAddr, net::SocketAddr,
@ -3080,20 +3080,17 @@ mod tests {
let keypair1 = Keypair::new(); let keypair1 = Keypair::new();
let success_tx = let success_tx =
system_transaction::transfer(&mint_keypair, &pubkey, 1, genesis_config.hash()); system_transaction::transfer(&mint_keypair, &pubkey, 0, genesis_config.hash());
let success_signature = success_tx.signatures[0]; let success_signature = success_tx.signatures[0];
let entry_1 = next_entry(&genesis_config.hash(), 1, vec![success_tx.clone()]); let entry_1 = next_entry(&genesis_config.hash(), 1, vec![success_tx.clone()]);
let ix_error_tx = let ix_error_tx =
system_transaction::transfer(&keypair1, &pubkey1, 10, genesis_config.hash()); system_transaction::transfer(&keypair1, &pubkey1, std::u64::MAX, genesis_config.hash());
let ix_error_signature = ix_error_tx.signatures[0]; let ix_error_signature = ix_error_tx.signatures[0];
let entry_2 = next_entry(&entry_1.hash, 1, vec![ix_error_tx.clone()]); let entry_2 = next_entry(&entry_1.hash, 1, vec![ix_error_tx.clone()]);
let fail_tx = let entries = vec![entry_1, entry_2];
system_transaction::transfer(&mint_keypair, &pubkey1, 1, genesis_config.hash());
let entry_3 = next_entry(&entry_2.hash, 1, vec![fail_tx.clone()]);
let entries = vec![entry_1, entry_2, entry_3];
let transactions = vec![success_tx.into(), ix_error_tx.into(), fail_tx.into()]; let transactions = vec![success_tx.into(), ix_error_tx.into()];
bank.transfer(4, &mint_keypair, &keypair1.pubkey()).unwrap(); bank.transfer(1, &mint_keypair, &keypair1.pubkey()).unwrap();
let start = Arc::new(Instant::now()); let start = Arc::new(Instant::now());
let working_bank = WorkingBank { let working_bank = WorkingBank {
@ -3155,27 +3152,24 @@ mod tests {
transaction_status_service.join().unwrap(); transaction_status_service.join().unwrap();
let confirmed_block = blockstore.get_rooted_block(bank.slot(), false).unwrap(); let confirmed_block = blockstore.get_rooted_block(bank.slot(), false).unwrap();
assert_eq!(confirmed_block.transactions.len(), 3); let actual_tx_results: Vec<_> = confirmed_block
.transactions
for TransactionWithStatusMeta { transaction, meta } in .into_iter()
confirmed_block.transactions.into_iter() .map(|TransactionWithMetadata { transaction, meta }| {
{ (transaction.signatures[0], meta.status)
if transaction.signatures[0] == success_signature { })
let meta = meta.unwrap(); .collect();
assert_eq!(meta.status, Ok(())); let expected_tx_results = vec![
} else if transaction.signatures[0] == ix_error_signature { (success_signature, Ok(())),
let meta = meta.unwrap(); (
assert_eq!( ix_error_signature,
meta.status,
Err(TransactionError::InstructionError( Err(TransactionError::InstructionError(
0, 0,
InstructionError::Custom(1) InstructionError::Custom(1),
)) )),
); ),
} else { ];
assert_eq!(meta, None); assert_eq!(actual_tx_results, expected_tx_results);
}
}
poh_recorder poh_recorder
.lock() .lock()

43
core/src/replay_stage.rs
View File

@ -2670,7 +2670,7 @@ mod tests {
transaction::TransactionError, transaction::TransactionError,
}, },
solana_streamer::socket::SocketAddrSpace, solana_streamer::socket::SocketAddrSpace,
solana_transaction_status::TransactionWithStatusMeta, solana_transaction_status::TransactionWithMetadata,
solana_vote_program::{ solana_vote_program::{
vote_state::{VoteState, VoteStateVersions}, vote_state::{VoteState, VoteStateVersions},
vote_transaction, vote_transaction,
@ -3472,36 +3472,37 @@ mod tests {
let bank1 = Arc::new(Bank::new_from_parent(&bank0, &Pubkey::default(), 1)); let bank1 = Arc::new(Bank::new_from_parent(&bank0, &Pubkey::default(), 1));
let slot = bank1.slot(); let slot = bank1.slot();
let signatures = create_test_transactions_and_populate_blockstore( let mut test_signatures_iter = create_test_transactions_and_populate_blockstore(
vec![&mint_keypair, &keypair1, &keypair2, &keypair3], vec![&mint_keypair, &keypair1, &keypair2, &keypair3],
bank0.slot(), bank0.slot(),
bank1, bank1,
blockstore.clone(), blockstore.clone(),
Arc::new(AtomicU64::default()), Arc::new(AtomicU64::default()),
); )
.into_iter();
let confirmed_block = blockstore.get_rooted_block(slot, false).unwrap(); let confirmed_block = blockstore.get_rooted_block(slot, false).unwrap();
assert_eq!(confirmed_block.transactions.len(), 3); let actual_tx_results: Vec<_> = confirmed_block
.transactions
.into_iter()
.map(|TransactionWithMetadata { transaction, meta }| {
(transaction.signatures[0], meta.status)
})
.collect();
for TransactionWithStatusMeta { transaction, meta } in let expected_tx_results = vec![
confirmed_block.transactions.into_iter() (test_signatures_iter.next().unwrap(), Ok(())),
{ (
if transaction.signatures[0] == signatures[0] { test_signatures_iter.next().unwrap(),
let meta = meta.unwrap();
assert_eq!(meta.status, Ok(()));
} else if transaction.signatures[0] == signatures[1] {
let meta = meta.unwrap();
assert_eq!(
meta.status,
Err(TransactionError::InstructionError( Err(TransactionError::InstructionError(
0, 0,
InstructionError::Custom(1) InstructionError::Custom(1),
)) )),
); ),
} else { ];
assert_eq!(meta, None);
} assert_eq!(actual_tx_results, expected_tx_results);
} assert!(test_signatures_iter.next().is_none());
} }
Blockstore::destroy(&ledger_path).unwrap(); Blockstore::destroy(&ledger_path).unwrap();
} }

16
ledger-tool/src/bigtable.rs
View File

@ -16,7 +16,9 @@ use {
}, },
solana_ledger::{blockstore::Blockstore, blockstore_db::AccessType}, solana_ledger::{blockstore::Blockstore, blockstore_db::AccessType},
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}, solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
solana_transaction_status::{ConfirmedBlock, EncodedTransaction, UiTransactionEncoding}, solana_transaction_status::{
ConfirmedBlockWithOptionalMetadata, EncodedTransaction, UiTransactionEncoding,
},
std::{ std::{
collections::HashSet, collections::HashSet,
path::Path, path::Path,
@ -30,7 +32,6 @@ async fn upload(
blockstore: Blockstore, blockstore: Blockstore,
starting_slot: Slot, starting_slot: Slot,
ending_slot: Option<Slot>, ending_slot: Option<Slot>,
allow_missing_metadata: bool,
force_reupload: bool, force_reupload: bool,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None) let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
@ -42,7 +43,6 @@ async fn upload(
bigtable, bigtable,
starting_slot, starting_slot,
ending_slot, ending_slot,
allow_missing_metadata,
force_reupload, force_reupload,
Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)),
) )
@ -194,7 +194,7 @@ pub async fn transaction_history(
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(true, None, None).await?; let bigtable = solana_storage_bigtable::LedgerStorage::new(true, None, None).await?;
let mut loaded_block: Option<(Slot, ConfirmedBlock)> = None; let mut loaded_block: Option<(Slot, ConfirmedBlockWithOptionalMetadata)> = None;
while limit > 0 { while limit > 0 {
let results = bigtable let results = bigtable
.get_confirmed_signatures_for_address( .get_confirmed_signatures_for_address(
@ -306,12 +306,6 @@ impl BigTableSubCommand for App<'_, '_> {
.index(2) .index(2)
.help("Stop uploading at this slot [default: last available slot]"), .help("Stop uploading at this slot [default: last available slot]"),
) )
.arg(
Arg::with_name("allow_missing_metadata")
.long("allow-missing-metadata")
.takes_value(false)
.help("Don't panic if transaction metadata is missing"),
)
.arg( .arg(
Arg::with_name("force_reupload") Arg::with_name("force_reupload")
.long("force") .long("force")
@ -506,7 +500,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
("upload", Some(arg_matches)) => { ("upload", Some(arg_matches)) => {
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0); let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok(); let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
let force_reupload = arg_matches.is_present("force_reupload"); let force_reupload = arg_matches.is_present("force_reupload");
let blockstore = crate::open_blockstore( let blockstore = crate::open_blockstore(
&canonicalize_ledger_path(ledger_path), &canonicalize_ledger_path(ledger_path),
@ -518,7 +511,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
blockstore, blockstore,
starting_slot, starting_slot,
ending_slot, ending_slot,
allow_missing_metadata,
force_reupload, force_reupload,
)) ))
} }

16
ledger/src/bigtable_upload.rs
View File

@ -25,7 +25,6 @@ pub async fn upload_confirmed_blocks(
bigtable: solana_storage_bigtable::LedgerStorage, bigtable: solana_storage_bigtable::LedgerStorage,
starting_slot: Slot, starting_slot: Slot,
ending_slot: Option<Slot>, ending_slot: Option<Slot>,
allow_missing_metadata: bool,
force_reupload: bool, force_reupload: bool,
exit: Arc<AtomicBool>, exit: Arc<AtomicBool>,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
@ -186,20 +185,7 @@ pub async fn upload_confirmed_blocks(
num_blocks -= 1; num_blocks -= 1;
None None
} }
Some(confirmed_block) => { Some(confirmed_block) => Some(bigtable.upload_confirmed_block(slot, confirmed_block)),
if confirmed_block
.transactions
.iter()
.any(|transaction| transaction.meta.is_none())
{
if allow_missing_metadata {
info!("Transaction metadata missing from slot {}", slot);
} else {
panic!("Transaction metadata missing from slot {}", slot);
}
}
Some(bigtable.upload_confirmed_block(slot, confirmed_block))
}
}); });
for result in futures::future::join_all(uploads).await { for result in futures::future::join_all(uploads).await {

1
ledger/src/bigtable_upload_service.rs
View File

@ -72,7 +72,6 @@ impl BigTableUploadService {
bigtable_ledger_storage.clone(), bigtable_ledger_storage.clone(),
start_slot, start_slot,
Some(end_slot), Some(end_slot),
true,
false, false,
exit.clone(), exit.clone(),
)); ));

56
ledger/src/blockstore.rs
View File

@ -42,7 +42,7 @@ use {
solana_storage_proto::{StoredExtendedRewards, StoredTransactionStatusMeta}, solana_storage_proto::{StoredExtendedRewards, StoredTransactionStatusMeta},
solana_transaction_status::{ solana_transaction_status::{
ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards, ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards,
TransactionStatusMeta, TransactionWithStatusMeta, TransactionStatusMeta, TransactionWithMetadata, TransactionWithOptionalMetadata,
}, },
std::{ std::{
borrow::Cow, borrow::Cow,
@ -1950,7 +1950,7 @@ impl Blockstore {
// from shreds received. // from shreds received.
parent_slot: slot_meta.parent_slot.unwrap(), parent_slot: slot_meta.parent_slot.unwrap(),
transactions: self transactions: self
.map_transactions_to_statuses(slot, slot_transaction_iterator), .map_transactions_to_statuses(slot, slot_transaction_iterator)?,
rewards, rewards,
block_time, block_time,
block_height, block_height,
@ -1965,17 +1965,16 @@ impl Blockstore {
&self, &self,
slot: Slot, slot: Slot,
iterator: impl Iterator<Item = Transaction> + 'a, iterator: impl Iterator<Item = Transaction> + 'a,
) -> Vec<TransactionWithStatusMeta> { ) -> Result<Vec<TransactionWithMetadata>> {
iterator iterator
.map(|transaction| { .map(|transaction| {
let signature = transaction.signatures[0]; let signature = transaction.signatures[0];
TransactionWithStatusMeta { Ok(TransactionWithMetadata {
transaction, transaction,
meta: self meta: self
.read_transaction_status((signature, slot)) .read_transaction_status((signature, slot))?
.ok() .ok_or(BlockstoreError::MissingTransactionMetadata)?,
.flatten(), })
}
}) })
.collect() .collect()
} }
@ -2256,7 +2255,7 @@ impl Blockstore {
let block_time = self.get_block_time(slot)?; let block_time = self.get_block_time(slot)?;
Ok(Some(ConfirmedTransaction { Ok(Some(ConfirmedTransaction {
slot, slot,
transaction: TransactionWithStatusMeta { transaction: TransactionWithOptionalMetadata {
transaction, transaction,
meta: Some(status), meta: Some(status),
}, },
@ -6082,7 +6081,7 @@ pub mod tests {
.put_meta_bytes(slot - 1, &serialize(&parent_meta).unwrap()) .put_meta_bytes(slot - 1, &serialize(&parent_meta).unwrap())
.unwrap(); .unwrap();
let expected_transactions: Vec<TransactionWithStatusMeta> = entries let expected_transactions: Vec<TransactionWithMetadata> = entries
.iter() .iter()
.cloned() .cloned()
.filter(|entry| !entry.is_tick()) .filter(|entry| !entry.is_tick())
@ -6143,9 +6142,9 @@ pub mod tests {
.transaction_status_cf .transaction_status_cf
.put_protobuf((0, signature, slot + 2), &status) .put_protobuf((0, signature, slot + 2), &status)
.unwrap(); .unwrap();
TransactionWithStatusMeta { TransactionWithMetadata {
transaction, transaction,
meta: Some(TransactionStatusMeta { meta: TransactionStatusMeta {
status: Ok(()), status: Ok(()),
fee: 42, fee: 42,
pre_balances, pre_balances,
@ -6155,7 +6154,7 @@ pub mod tests {
pre_token_balances: Some(vec![]), pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]), post_token_balances: Some(vec![]),
rewards: Some(vec![]), rewards: Some(vec![]),
}), },
} }
}) })
.collect(); .collect();
@ -6952,7 +6951,7 @@ pub mod tests {
blockstore.insert_shreds(shreds, None, false).unwrap(); blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.set_roots(vec![slot - 1, slot].iter()).unwrap(); blockstore.set_roots(vec![slot - 1, slot].iter()).unwrap();
let expected_transactions: Vec<TransactionWithStatusMeta> = entries let expected_transactions: Vec<TransactionWithOptionalMetadata> = entries
.iter() .iter()
.cloned() .cloned()
.filter(|entry| !entry.is_tick()) .filter(|entry| !entry.is_tick())
@ -6989,7 +6988,7 @@ pub mod tests {
.transaction_status_cf .transaction_status_cf
.put_protobuf((0, signature, slot), &status) .put_protobuf((0, signature, slot), &status)
.unwrap(); .unwrap();
TransactionWithStatusMeta { TransactionWithOptionalMetadata {
transaction, transaction,
meta: Some(TransactionStatusMeta { meta: Some(TransactionStatusMeta {
status: Ok(()), status: Ok(()),
@ -7030,7 +7029,7 @@ pub mod tests {
blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap(); blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap();
*blockstore.lowest_cleanup_slot.write().unwrap() = slot; *blockstore.lowest_cleanup_slot.write().unwrap() = slot;
for TransactionWithStatusMeta { transaction, .. } in expected_transactions { for TransactionWithOptionalMetadata { transaction, .. } in expected_transactions {
let signature = transaction.signatures[0]; let signature = transaction.signatures[0];
assert_eq!(blockstore.get_rooted_transaction(signature).unwrap(), None,); assert_eq!(blockstore.get_rooted_transaction(signature).unwrap(), None,);
assert_eq!( assert_eq!(
@ -7051,7 +7050,7 @@ pub mod tests {
let blockstore = Blockstore::open(&ledger_path).unwrap(); let blockstore = Blockstore::open(&ledger_path).unwrap();
blockstore.insert_shreds(shreds, None, false).unwrap(); blockstore.insert_shreds(shreds, None, false).unwrap();
let expected_transactions: Vec<TransactionWithStatusMeta> = entries let expected_transactions: Vec<TransactionWithOptionalMetadata> = entries
.iter() .iter()
.cloned() .cloned()
.filter(|entry| !entry.is_tick()) .filter(|entry| !entry.is_tick())
@ -7088,7 +7087,7 @@ pub mod tests {
.transaction_status_cf .transaction_status_cf
.put_protobuf((0, signature, slot), &status) .put_protobuf((0, signature, slot), &status)
.unwrap(); .unwrap();
TransactionWithStatusMeta { TransactionWithOptionalMetadata {
transaction, transaction,
meta: Some(TransactionStatusMeta { meta: Some(TransactionStatusMeta {
status: Ok(()), status: Ok(()),
@ -7122,7 +7121,7 @@ pub mod tests {
blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap(); blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap();
*blockstore.lowest_cleanup_slot.write().unwrap() = slot; *blockstore.lowest_cleanup_slot.write().unwrap() = slot;
for TransactionWithStatusMeta { transaction, .. } in expected_transactions { for TransactionWithOptionalMetadata { transaction, .. } in expected_transactions {
let signature = transaction.signatures[0]; let signature = transaction.signatures[0];
assert_eq!( assert_eq!(
blockstore blockstore
@ -7877,6 +7876,16 @@ pub mod tests {
.unwrap(); .unwrap();
transactions.push(transaction); transactions.push(transaction);
} }
let map_result =
blockstore.map_transactions_to_statuses(slot, transactions.clone().into_iter());
assert!(map_result.is_ok());
let map = map_result.unwrap();
assert_eq!(map.len(), 4);
for (x, m) in map.iter().enumerate() {
assert_eq!(m.meta.fee, x as u64);
}
// Push transaction that will not have matching status, as a test case // Push transaction that will not have matching status, as a test case
transactions.push(Transaction::new_with_compiled_instructions( transactions.push(Transaction::new_with_compiled_instructions(
&[&Keypair::new()], &[&Keypair::new()],
@ -7886,12 +7895,9 @@ pub mod tests {
vec![CompiledInstruction::new(1, &(), vec![0])], vec![CompiledInstruction::new(1, &(), vec![0])],
)); ));
let map = blockstore.map_transactions_to_statuses(slot, transactions.into_iter()); let map_result =
assert_eq!(map.len(), 5); blockstore.map_transactions_to_statuses(slot, transactions.into_iter());
for (x, m) in map.iter().take(4).enumerate() { assert_matches!(map_result, Err(BlockstoreError::MissingTransactionMetadata));
assert_eq!(m.meta.as_ref().unwrap().fee, x as u64);
}
assert_eq!(map[4].meta, None);
} }
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction"); Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
} }

1
ledger/src/blockstore_db.rs
View File

@ -101,6 +101,7 @@ pub enum BlockstoreError {
ProtobufDecodeError(#[from] prost::DecodeError), ProtobufDecodeError(#[from] prost::DecodeError),
ParentEntriesUnavailable, ParentEntriesUnavailable,
SlotUnavailable, SlotUnavailable,
MissingTransactionMetadata,
} }
pub type Result<T> = std::result::Result<T, BlockstoreError>; pub type Result<T> = std::result::Result<T, BlockstoreError>;

4
local-cluster/tests/common.rs
View File

@ -250,6 +250,10 @@ pub fn run_cluster_partition<C>(
..ValidatorConfig::default_for_test() ..ValidatorConfig::default_for_test()
}; };
// Enabled so that transaction statuses are written to blockstore which enables
// tests to detect when a block has been replayed.
validator_config.rpc_config.enable_rpc_transaction_history = true;
// Returns: // Returns:
// 1) The keys for the validators // 1) The keys for the validators
// 2) The amount of time it would take to iterate through one full iteration of the given // 2) The amount of time it would take to iterate through one full iteration of the given

12
local-cluster/tests/local_cluster_slow.rs
View File

@ -725,13 +725,19 @@ fn find_latest_replayed_slot_from_ledger(
// Wait for the slot to be replayed // Wait for the slot to be replayed
loop { loop {
info!("Waiting for slot {} to be replayed", latest_slot); info!("Waiting for slot {} to be replayed", latest_slot);
if !blockstore let replayed_transactions = blockstore
.map_transactions_to_statuses( .map_transactions_to_statuses(
latest_slot, latest_slot,
non_tick_entry.transactions.clone().into_iter(), non_tick_entry.transactions.clone().into_iter(),
) )
.is_empty() .unwrap_or_else(|_| {
{ info!(
"Transaction statuses for slot {} haven't been written yet",
latest_slot
);
Vec::new()
});
if !replayed_transactions.is_empty() {
return ( return (
latest_slot, latest_slot,
AncestorIterator::new(latest_slot, &blockstore).collect(), AncestorIterator::new(latest_slot, &blockstore).collect(),

4
programs/bpf/tests/programs.rs
View File

@ -50,7 +50,7 @@ use solana_sdk::{
}; };
use solana_transaction_status::{ use solana_transaction_status::{
token_balances::collect_token_balances, ConfirmedTransaction, InnerInstructions, token_balances::collect_token_balances, ConfirmedTransaction, InnerInstructions,
TransactionStatusMeta, TransactionWithStatusMeta, UiTransactionEncoding, TransactionStatusMeta, TransactionWithOptionalMetadata, UiTransactionEncoding,
}; };
use std::{ use std::{
cell::RefCell, collections::HashMap, env, fs::File, io::Read, path::PathBuf, str::FromStr, cell::RefCell, collections::HashMap, env, fs::File, io::Read, path::PathBuf, str::FromStr,
@ -403,7 +403,7 @@ fn execute_transactions(bank: &Bank, txs: &[Transaction]) -> Vec<ConfirmedTransa
ConfirmedTransaction { ConfirmedTransaction {
slot: bank.slot(), slot: bank.slot(),
transaction: TransactionWithStatusMeta { transaction: TransactionWithOptionalMetadata {
transaction: tx.clone(), transaction: tx.clone(),
meta: Some(tx_status_meta), meta: Some(tx_status_meta),
}, },

38
rpc/src/rpc.rs
View File

@ -73,9 +73,9 @@ use {
solana_storage_bigtable::Error as StorageError, solana_storage_bigtable::Error as StorageError,
solana_streamer::socket::SocketAddrSpace, solana_streamer::socket::SocketAddrSpace,
solana_transaction_status::{ solana_transaction_status::{
ConfirmedBlock, ConfirmedTransactionStatusWithSignature, EncodedConfirmedTransaction, ConfirmedBlockWithOptionalMetadata, ConfirmedTransactionStatusWithSignature,
Reward, RewardType, TransactionConfirmationStatus, TransactionStatus, UiConfirmedBlock, EncodedConfirmedTransaction, Reward, RewardType, TransactionConfirmationStatus,
UiTransactionEncoding, TransactionStatus, UiConfirmedBlock, UiTransactionEncoding,
}, },
solana_vote_program::vote_state::{VoteState, MAX_LOCKOUT_HISTORY}, solana_vote_program::vote_state::{VoteState, MAX_LOCKOUT_HISTORY},
spl_token::{ spl_token::{
@ -990,7 +990,7 @@ impl JsonRpcRequestProcessor {
self.check_status_is_complete(slot)?; self.check_status_is_complete(slot)?;
let result = self.blockstore.get_rooted_block(slot, true); let result = self.blockstore.get_rooted_block(slot, true);
self.check_blockstore_root(&result, slot)?; self.check_blockstore_root(&result, slot)?;
let configure_block = |confirmed_block: ConfirmedBlock| { let configure_block = |confirmed_block: ConfirmedBlockWithOptionalMetadata| {
let mut confirmed_block = let mut confirmed_block =
confirmed_block.configure(encoding, transaction_details, show_rewards); confirmed_block.configure(encoding, transaction_details, show_rewards);
if slot == 0 { if slot == 0 {
@ -1008,7 +1008,10 @@ impl JsonRpcRequestProcessor {
} }
} }
self.check_slot_cleaned_up(&result, slot)?; self.check_slot_cleaned_up(&result, slot)?;
return Ok(result.ok().map(configure_block)); return Ok(result
.ok()
.map(ConfirmedBlockWithOptionalMetadata::from)
.map(configure_block));
} else if commitment.is_confirmed() { } else if commitment.is_confirmed() {
// Check if block is confirmed // Check if block is confirmed
let confirmed_bank = self.bank(Some(CommitmentConfig::confirmed())); let confirmed_bank = self.bank(Some(CommitmentConfig::confirmed()));
@ -1030,7 +1033,11 @@ impl JsonRpcRequestProcessor {
} }
} }
} }
confirmed_block.configure(encoding, transaction_details, show_rewards) ConfirmedBlockWithOptionalMetadata::from(confirmed_block).configure(
encoding,
transaction_details,
show_rewards,
)
})); }));
} }
} }
@ -4056,15 +4063,7 @@ pub fn create_test_transactions_and_populate_blockstore(
solana_sdk::system_transaction::transfer(keypair2, &keypair3.pubkey(), 10, blockhash); solana_sdk::system_transaction::transfer(keypair2, &keypair3.pubkey(), 10, blockhash);
let ix_error_signature = ix_error_tx.signatures[0]; let ix_error_signature = ix_error_tx.signatures[0];
let entry_2 = solana_ledger::entry::next_entry(&entry_1.hash, 1, vec![ix_error_tx]); let entry_2 = solana_ledger::entry::next_entry(&entry_1.hash, 1, vec![ix_error_tx]);
// Failed transaction let mut entries = vec![entry_1, entry_2];
let fail_tx = solana_sdk::system_transaction::transfer(
mint_keypair,
&keypair2.pubkey(),
2,
Hash::default(),
);
let entry_3 = solana_ledger::entry::next_entry(&entry_2.hash, 1, vec![fail_tx]);
let mut entries = vec![entry_1, entry_2, entry_3];
let shreds = solana_ledger::blockstore::entries_to_test_shreds( let shreds = solana_ledger::blockstore::entries_to_test_shreds(
entries.clone(), entries.clone(),
@ -4088,7 +4087,8 @@ pub fn create_test_transactions_and_populate_blockstore(
// Check that process_entries successfully writes can_commit transactions statuses, and // Check that process_entries successfully writes can_commit transactions statuses, and
// that they are matched properly by get_rooted_block // that they are matched properly by get_rooted_block
let _result = solana_ledger::blockstore_processor::process_entries( assert_eq!(
solana_ledger::blockstore_processor::process_entries(
&bank, &bank,
&mut entries, &mut entries,
true, true,
@ -4099,6 +4099,8 @@ pub fn create_test_transactions_and_populate_blockstore(
}, },
), ),
Some(&replay_vote_sender), Some(&replay_vote_sender),
),
Ok(())
); );
transaction_status_service.join().unwrap(); transaction_status_service.join().unwrap();
@ -6264,7 +6266,7 @@ pub mod tests {
let confirmed_block: Option<EncodedConfirmedBlock> = let confirmed_block: Option<EncodedConfirmedBlock> =
serde_json::from_value(result["result"].clone()).unwrap(); serde_json::from_value(result["result"].clone()).unwrap();
let confirmed_block = confirmed_block.unwrap(); let confirmed_block = confirmed_block.unwrap();
assert_eq!(confirmed_block.transactions.len(), 3); assert_eq!(confirmed_block.transactions.len(), 2);
assert_eq!(confirmed_block.rewards, vec![]); assert_eq!(confirmed_block.rewards, vec![]);
for EncodedTransactionWithStatusMeta { transaction, meta } in for EncodedTransactionWithStatusMeta { transaction, meta } in
@ -6309,7 +6311,7 @@ pub mod tests {
let confirmed_block: Option<EncodedConfirmedBlock> = let confirmed_block: Option<EncodedConfirmedBlock> =
serde_json::from_value(result["result"].clone()).unwrap(); serde_json::from_value(result["result"].clone()).unwrap();
let confirmed_block = confirmed_block.unwrap(); let confirmed_block = confirmed_block.unwrap();
assert_eq!(confirmed_block.transactions.len(), 3); assert_eq!(confirmed_block.transactions.len(), 2);
assert_eq!(confirmed_block.rewards, vec![]); assert_eq!(confirmed_block.rewards, vec![]);
for EncodedTransactionWithStatusMeta { transaction, meta } in for EncodedTransactionWithStatusMeta { transaction, meta } in

33
storage-bigtable/src/bigtable.rs
View File

@ -793,17 +793,42 @@ mod tests {
solana_sdk::{hash::Hash, signature::Keypair, system_transaction}, solana_sdk::{hash::Hash, signature::Keypair, system_transaction},
solana_storage_proto::convert::generated, solana_storage_proto::convert::generated,
solana_transaction_status::{ solana_transaction_status::{
ConfirmedBlock, TransactionStatusMeta, TransactionWithStatusMeta, ConfirmedBlockWithOptionalMetadata, TransactionStatusMeta,
TransactionWithOptionalMetadata,
}, },
std::convert::TryInto, std::convert::TryInto,
}; };
fn convert_from_block_with_optional_meta(
confirmed_block: ConfirmedBlockWithOptionalMetadata,
) -> generated::ConfirmedBlock {
let ConfirmedBlockWithOptionalMetadata {
previous_blockhash,
blockhash,
parent_slot,
transactions,
rewards,
block_time,
block_height,
} = confirmed_block;
generated::ConfirmedBlock {
previous_blockhash,
blockhash,
parent_slot,
transactions: transactions.into_iter().map(|tx| tx.into()).collect(),
rewards: rewards.into_iter().map(|r| r.into()).collect(),
block_time: block_time.map(|timestamp| generated::UnixTimestamp { timestamp }),
block_height: block_height.map(|block_height| generated::BlockHeight { block_height }),
}
}
#[test] #[test]
fn test_deserialize_protobuf_or_bincode_cell_data() { fn test_deserialize_protobuf_or_bincode_cell_data() {
let from = Keypair::new(); let from = Keypair::new();
let recipient = solana_sdk::pubkey::new_rand(); let recipient = solana_sdk::pubkey::new_rand();
let transaction = system_transaction::transfer(&from, &recipient, 42, Hash::default()); let transaction = system_transaction::transfer(&from, &recipient, 42, Hash::default());
let with_meta = TransactionWithStatusMeta { let with_meta = TransactionWithOptionalMetadata {
transaction, transaction,
meta: Some(TransactionStatusMeta { meta: Some(TransactionStatusMeta {
status: Ok(()), status: Ok(()),
@ -817,7 +842,7 @@ mod tests {
rewards: Some(vec![]), rewards: Some(vec![]),
}), }),
}; };
let block = ConfirmedBlock { let block = ConfirmedBlockWithOptionalMetadata {
transactions: vec![with_meta], transactions: vec![with_meta],
parent_slot: 1, parent_slot: 1,
blockhash: Hash::default().to_string(), blockhash: Hash::default().to_string(),
@ -831,7 +856,7 @@ mod tests {
) )
.unwrap(); .unwrap();
let protobuf_block = generated::ConfirmedBlock::from(block.clone()); let protobuf_block = convert_from_block_with_optional_meta(block.clone());
let mut buf = Vec::with_capacity(protobuf_block.encoded_len()); let mut buf = Vec::with_capacity(protobuf_block.encoded_len());
protobuf_block.encode(&mut buf).unwrap(); protobuf_block.encode(&mut buf).unwrap();
let protobuf_block = compress_best(&buf).unwrap(); let protobuf_block = compress_best(&buf).unwrap();

33
storage-bigtable/src/lib.rs
View File

@ -13,10 +13,10 @@ use {
}, },
solana_storage_proto::convert::{generated, tx_by_addr}, solana_storage_proto::convert::{generated, tx_by_addr},
solana_transaction_status::{ solana_transaction_status::{
extract_and_fmt_memos, ConfirmedBlock, ConfirmedTransaction, extract_and_fmt_memos, ConfirmedBlock, ConfirmedBlockWithOptionalMetadata,
ConfirmedTransactionStatusWithSignature, Reward, TransactionByAddrInfo, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Reward,
TransactionConfirmationStatus, TransactionStatus, TransactionStatusMeta, TransactionByAddrInfo, TransactionConfirmationStatus, TransactionStatus,
TransactionWithStatusMeta, TransactionStatusMeta, TransactionWithMetadata, TransactionWithOptionalMetadata,
}, },
std::{ std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
@ -114,9 +114,9 @@ struct StoredConfirmedBlock {
block_height: Option<u64>, block_height: Option<u64>,
} }
impl From<ConfirmedBlock> for StoredConfirmedBlock { impl From<ConfirmedBlockWithOptionalMetadata> for StoredConfirmedBlock {
fn from(confirmed_block: ConfirmedBlock) -> Self { fn from(confirmed_block: ConfirmedBlockWithOptionalMetadata) -> Self {
let ConfirmedBlock { let ConfirmedBlockWithOptionalMetadata {
previous_blockhash, previous_blockhash,
blockhash, blockhash,
parent_slot, parent_slot,
@ -138,7 +138,7 @@ impl From<ConfirmedBlock> for StoredConfirmedBlock {
} }
} }
impl From<StoredConfirmedBlock> for ConfirmedBlock { impl From<StoredConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
fn from(confirmed_block: StoredConfirmedBlock) -> Self { fn from(confirmed_block: StoredConfirmedBlock) -> Self {
let StoredConfirmedBlock { let StoredConfirmedBlock {
previous_blockhash, previous_blockhash,
@ -168,8 +168,8 @@ struct StoredConfirmedBlockTransaction {
meta: Option<StoredConfirmedBlockTransactionStatusMeta>, meta: Option<StoredConfirmedBlockTransactionStatusMeta>,
} }
impl From<TransactionWithStatusMeta> for StoredConfirmedBlockTransaction { impl From<TransactionWithOptionalMetadata> for StoredConfirmedBlockTransaction {
fn from(value: TransactionWithStatusMeta) -> Self { fn from(value: TransactionWithOptionalMetadata) -> Self {
Self { Self {
transaction: value.transaction, transaction: value.transaction,
meta: value.meta.map(|meta| meta.into()), meta: value.meta.map(|meta| meta.into()),
@ -177,7 +177,7 @@ impl From<TransactionWithStatusMeta> for StoredConfirmedBlockTransaction {
} }
} }
impl From<StoredConfirmedBlockTransaction> for TransactionWithStatusMeta { impl From<StoredConfirmedBlockTransaction> for TransactionWithOptionalMetadata {
fn from(value: StoredConfirmedBlockTransaction) -> Self { fn from(value: StoredConfirmedBlockTransaction) -> Self {
Self { Self {
transaction: value.transaction, transaction: value.transaction,
@ -392,7 +392,10 @@ impl LedgerStorage {
} }
/// Fetch the confirmed block from the desired slot /// Fetch the confirmed block from the desired slot
pub async fn get_confirmed_block(&self, slot: Slot) -> Result<ConfirmedBlock> { pub async fn get_confirmed_block(
&self,
slot: Slot,
) -> Result<ConfirmedBlockWithOptionalMetadata> {
debug!( debug!(
"LedgerStorage::get_confirmed_block request received: {:?}", "LedgerStorage::get_confirmed_block request received: {:?}",
slot slot
@ -635,8 +638,8 @@ impl LedgerStorage {
let mut tx_cells = vec![]; let mut tx_cells = vec![];
for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() { for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() {
let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta; let TransactionWithMetadata { meta, transaction } = transaction_with_meta;
let err = meta.as_ref().and_then(|meta| meta.status.clone().err()); let err = meta.status.clone().err();
let index = index as u32; let index = index as u32;
let signature = transaction.signatures[0]; let signature = transaction.signatures[0];
let memo = extract_and_fmt_memos(&transaction.message); let memo = extract_and_fmt_memos(&transaction.message);
@ -723,7 +726,7 @@ impl LedgerStorage {
let mut expected_tx_infos: HashMap<String, UploadedTransaction> = HashMap::new(); let mut expected_tx_infos: HashMap<String, UploadedTransaction> = HashMap::new();
let confirmed_block = self.get_confirmed_block(slot).await?; let confirmed_block = self.get_confirmed_block(slot).await?;
for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() { for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() {
let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta; let TransactionWithOptionalMetadata { meta, transaction } = transaction_with_meta;
let signature = transaction.signatures[0]; let signature = transaction.signatures[0];
let index = index as u32; let index = index as u32;
let err = meta.as_ref().and_then(|meta| meta.status.clone().err()); let err = meta.as_ref().and_then(|meta| meta.status.clone().err());

28
storage-proto/src/convert.rs
View File

@ -10,8 +10,9 @@ use {
transaction::{Transaction, TransactionError}, transaction::{Transaction, TransactionError},
}, },
solana_transaction_status::{ solana_transaction_status::{
ConfirmedBlock, InnerInstructions, Reward, RewardType, TransactionByAddrInfo, ConfirmedBlock, ConfirmedBlockWithOptionalMetadata, InnerInstructions, Reward, RewardType,
TransactionStatusMeta, TransactionTokenBalance, TransactionWithStatusMeta, TransactionByAddrInfo, TransactionStatusMeta, TransactionTokenBalance,
TransactionWithMetadata, TransactionWithOptionalMetadata,
}, },
std::{ std::{
convert::{TryFrom, TryInto}, convert::{TryFrom, TryInto},
@ -135,7 +136,7 @@ impl From<ConfirmedBlock> for generated::ConfirmedBlock {
} }
} }
impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlock { impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
type Error = bincode::Error; type Error = bincode::Error;
fn try_from( fn try_from(
confirmed_block: generated::ConfirmedBlock, confirmed_block: generated::ConfirmedBlock,
@ -157,7 +158,8 @@ impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlock {
transactions: transactions transactions: transactions
.into_iter() .into_iter()
.map(|tx| tx.try_into()) .map(|tx| tx.try_into())
.collect::<std::result::Result<Vec<TransactionWithStatusMeta>, Self::Error>>()?, .collect::<std::result::Result<Vec<TransactionWithOptionalMetadata>, Self::Error>>(
)?,
rewards: rewards.into_iter().map(|r| r.into()).collect(), rewards: rewards.into_iter().map(|r| r.into()).collect(),
block_time: block_time.map(|generated::UnixTimestamp { timestamp }| timestamp), block_time: block_time.map(|generated::UnixTimestamp { timestamp }| timestamp),
block_height: block_height.map(|generated::BlockHeight { block_height }| block_height), block_height: block_height.map(|generated::BlockHeight { block_height }| block_height),
@ -165,17 +167,25 @@ impl TryFrom<generated::ConfirmedBlock> for ConfirmedBlock {
} }
} }
impl From<TransactionWithStatusMeta> for generated::ConfirmedTransaction { impl From<TransactionWithOptionalMetadata> for generated::ConfirmedTransaction {
fn from(value: TransactionWithStatusMeta) -> Self { fn from(value: TransactionWithOptionalMetadata) -> Self {
let meta = value.meta.map(|meta| meta.into());
Self { Self {
transaction: Some(value.transaction.into()), transaction: Some(value.transaction.into()),
meta, meta: value.meta.map(|meta| meta.into()),
} }
} }
} }
impl TryFrom<generated::ConfirmedTransaction> for TransactionWithStatusMeta { impl From<TransactionWithMetadata> for generated::ConfirmedTransaction {
fn from(value: TransactionWithMetadata) -> Self {
Self {
transaction: Some(value.transaction.into()),
meta: Some(value.meta.into()),
}
}
}
impl TryFrom<generated::ConfirmedTransaction> for TransactionWithOptionalMetadata {
type Error = bincode::Error; type Error = bincode::Error;
fn try_from(value: generated::ConfirmedTransaction) -> std::result::Result<Self, Self::Error> { fn try_from(value: generated::ConfirmedTransaction) -> std::result::Result<Self, Self::Error> {
let meta = value.meta.map(|meta| meta.try_into()).transpose()?; let meta = value.meta.map(|meta| meta.try_into()).transpose()?;

60
transaction-status/src/lib.rs
View File

@ -355,19 +355,52 @@ pub struct Reward {
pub type Rewards = Vec<Reward>; pub type Rewards = Vec<Reward>;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ConfirmedBlock { pub struct ConfirmedBlock {
pub previous_blockhash: String, pub previous_blockhash: String,
pub blockhash: String, pub blockhash: String,
pub parent_slot: Slot, pub parent_slot: Slot,
pub transactions: Vec<TransactionWithStatusMeta>, pub transactions: Vec<TransactionWithMetadata>,
pub rewards: Rewards, pub rewards: Rewards,
pub block_time: Option<UnixTimestamp>, pub block_time: Option<UnixTimestamp>,
pub block_height: Option<u64>, pub block_height: Option<u64>,
} }
impl ConfirmedBlock { #[derive(Clone, Debug, PartialEq)]
pub struct ConfirmedBlockWithOptionalMetadata {
pub previous_blockhash: String,
pub blockhash: String,
pub parent_slot: Slot,
pub transactions: Vec<TransactionWithOptionalMetadata>,
pub rewards: Rewards,
pub block_time: Option<UnixTimestamp>,
pub block_height: Option<u64>,
}
impl From<ConfirmedBlock> for ConfirmedBlockWithOptionalMetadata {
fn from(block: ConfirmedBlock) -> Self {
Self {
previous_blockhash: block.previous_blockhash,
blockhash: block.blockhash,
parent_slot: block.parent_slot,
transactions: block
.transactions
.into_iter()
.map(|TransactionWithMetadata { transaction, meta }| {
TransactionWithOptionalMetadata {
transaction,
meta: Some(meta),
}
})
.collect(),
rewards: block.rewards,
block_time: block.block_time,
block_height: block.block_height,
}
}
}
impl ConfirmedBlockWithOptionalMetadata {
pub fn encode(self, encoding: UiTransactionEncoding) -> EncodedConfirmedBlock { pub fn encode(self, encoding: UiTransactionEncoding) -> EncodedConfirmedBlock {
EncodedConfirmedBlock { EncodedConfirmedBlock {
previous_blockhash: self.previous_blockhash, previous_blockhash: self.previous_blockhash,
@ -499,12 +532,10 @@ impl Default for TransactionDetails {
} }
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ConfirmedTransaction { pub struct ConfirmedTransaction {
pub slot: Slot, pub slot: Slot,
#[serde(flatten)] pub transaction: TransactionWithOptionalMetadata,
pub transaction: TransactionWithStatusMeta,
pub block_time: Option<UnixTimestamp>, pub block_time: Option<UnixTimestamp>,
} }
@ -561,14 +592,19 @@ pub struct UiParsedMessage {
pub instructions: Vec<UiInstruction>, pub instructions: Vec<UiInstruction>,
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq)]
#[serde(rename_all = "camelCase")] pub struct TransactionWithOptionalMetadata {
pub struct TransactionWithStatusMeta {
pub transaction: Transaction, pub transaction: Transaction,
pub meta: Option<TransactionStatusMeta>, pub meta: Option<TransactionStatusMeta>,
} }
impl TransactionWithStatusMeta { #[derive(Clone, Debug, PartialEq)]
pub struct TransactionWithMetadata {
pub transaction: Transaction,
pub meta: TransactionStatusMeta,
}
impl TransactionWithOptionalMetadata {
fn encode(self, encoding: UiTransactionEncoding) -> EncodedTransactionWithStatusMeta { fn encode(self, encoding: UiTransactionEncoding) -> EncodedTransactionWithStatusMeta {
let message = self.transaction.message(); let message = self.transaction.message();
let meta = self.meta.map(|meta| meta.encode(encoding, message)); let meta = self.meta.map(|meta| meta.encode(encoding, message));