gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert blockstore TransactionStatus column family to protobufs (#15733)

Browse Source 
* Prevent panic if TransactionStatus can't be deserialized

* Convert Blockstore TransactionStatus column to protobuf

* Add compatability test
This commit is contained in:
Tyera Eulberg
2021-03-05 09:05:35 -07:00
committed by GitHub
parent bd13262b42
commit 7e65289729
8 changed files with 383 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ledger/Cargo.toml
View File

@ -64,6 +64,7 @@ features = ["lz4"]
[dev-dependencies]
assert_matches = "1.3.0"
matches = "0.1.6"
solana-account-decoder = { path = "../account-decoder", version = "1.6.0" }
solana-budget-program = { path = "../programs/budget", version = "1.6.0" }
[build-dependencies]

324
ledger/src/blockstore.rs
View File

@ -35,7 +35,7 @@ use solana_sdk::{
timing::timestamp,
transaction::Transaction,
};
use solana_storage_proto::StoredExtendedRewards;
use solana_storage_proto::{StoredExtendedRewards, StoredTransactionStatusMeta};
use solana_transaction_status::{
ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards,
TransactionStatusMeta, TransactionWithStatusMeta,
@ -44,6 +44,7 @@ use std::{
cell::RefCell,
cmp,
collections::{HashMap, HashSet},
convert::TryInto,
fs,
io::{Error as IoError, ErrorKind},
path::{Path, PathBuf},
@ -1783,7 +1784,8 @@ impl Blockstore {
transaction,
meta: self
.read_transaction_status((signature, slot))
.expect("Expect database get to succeed"),
.ok()
.flatten(),
}
})
.collect()
@ -1799,10 +1801,9 @@ impl Blockstore {
self.transaction_status_index_cf
.put(1, &TransactionStatusIndexMeta::default())?;
// This dummy status improves compaction performance
self.transaction_status_cf.put(
cf::TransactionStatus::as_index(2),
&TransactionStatusMeta::default(),
)?;
let default_status = TransactionStatusMeta::default().into();
self.transaction_status_cf
.put_protobuf(cf::TransactionStatus::as_index(2), &default_status)?;
self.address_signatures_cf.put(
cf::AddressSignatures::as_index(2),
&AddressSignatureMeta::default(),
@ -1878,11 +1879,16 @@ impl Blockstore {
index: (Signature, Slot),
) -> Result<Option<TransactionStatusMeta>> {
let (signature, slot) = index;
let result = self.transaction_status_cf.get((0, signature, slot))?;
let result = self
.transaction_status_cf
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((0, signature, slot))?;
if result.is_none() {
Ok(self.transaction_status_cf.get((1, signature, slot))?)
Ok(self
.transaction_status_cf
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((1, signature, slot))?
.and_then(|meta| meta.try_into().ok()))
} else {
Ok(result)
Ok(result.and_then(|meta| meta.try_into().ok()))
}
}
@ -1892,15 +1898,16 @@ impl Blockstore {
signature: Signature,
writable_keys: Vec<&Pubkey>,
readonly_keys: Vec<&Pubkey>,
status: &TransactionStatusMeta,
status: TransactionStatusMeta,
) -> Result<()> {
let status = status.into();
// This write lock prevents interleaving issues with the transaction_status_index_cf by gating
// writes to that column
let mut w_active_transaction_status_index =
self.active_transaction_status_index.write().unwrap();
let primary_index = self.get_primary_index(slot, &mut w_active_transaction_status_index)?;
self.transaction_status_cf
.put((primary_index, signature, slot), status)?;
.put_protobuf((primary_index, signature, slot), &status)?;
for address in writable_keys {
self.address_signatures_cf.put(
(primary_index, *address, slot, signature),
@ -1928,14 +1935,18 @@ impl Blockstore {
(transaction_status_cf_primary_index, signature, 0),
IteratorDirection::Forward,
))?;
for ((i, sig, slot), data) in index_iterator {
for ((i, sig, slot), _data) in index_iterator {
counter += 1;
if i != transaction_status_cf_primary_index || sig != signature {
break;
}
if self.is_root(slot) {
let status: TransactionStatusMeta = deserialize(&data)?;
return Ok((Some((slot, status)), counter));
let status = self
.transaction_status_cf
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((i, sig, slot))?
.and_then(|status| status.try_into().ok())
.map(|status| (slot, status));
return Ok((status, counter));
}
}
}
@ -3531,6 +3542,7 @@ pub mod tests {
use bincode::serialize;
use itertools::Itertools;
use rand::{seq::SliceRandom, thread_rng};
use solana_account_decoder::parse_token::UiTokenAmount;
use solana_runtime::bank::{Bank, RewardType};
use solana_sdk::{
hash::{self, hash, Hash},
@ -3541,7 +3553,7 @@ pub mod tests {
transaction::TransactionError,
};
use solana_storage_proto::convert::generated;
use solana_transaction_status::{InnerInstructions, Reward, Rewards};
use solana_transaction_status::{InnerInstructions, Reward, Rewards, TransactionTokenBalance};
use std::time::Duration;
// used for tests only
@ -5706,37 +5718,35 @@ pub mod tests {
post_balances.push(i as u64 * 11);
}
let signature = transaction.signatures[0];
let status = TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: pre_balances.clone(),
post_balances: post_balances.clone(),
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
}
.into();
ledger
.transaction_status_cf
.put(
(0, signature, slot),
&TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: pre_balances.clone(),
post_balances: post_balances.clone(),
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
},
)
.put_protobuf((0, signature, slot), &status)
.unwrap();
let status = TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: pre_balances.clone(),
post_balances: post_balances.clone(),
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
}
.into();
ledger
.transaction_status_cf
.put(
(0, signature, slot + 1),
&TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: pre_balances.clone(),
post_balances: post_balances.clone(),
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
},
)
.put_protobuf((0, signature, slot + 1), &status)
.unwrap();
TransactionWithStatusMeta {
transaction,
@ -5826,27 +5836,30 @@ pub mod tests {
// result not found
assert!(transaction_status_cf
.get((0, Signature::default(), 0))
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((
0,
Signature::default(),
0
))
.unwrap()
.is_none());
// insert value
let status = TransactionStatusMeta {
status: solana_sdk::transaction::Result::<()>::Err(
TransactionError::AccountNotFound,
),
fee: 5u64,
pre_balances: pre_balances_vec.clone(),
post_balances: post_balances_vec.clone(),
inner_instructions: Some(inner_instructions_vec.clone()),
log_messages: Some(log_messages_vec.clone()),
pre_token_balances: Some(pre_token_balances_vec.clone()),
post_token_balances: Some(post_token_balances_vec.clone()),
}
.into();
assert!(transaction_status_cf
.put(
(0, Signature::default(), 0),
&TransactionStatusMeta {
status: solana_sdk::transaction::Result::<()>::Err(
TransactionError::AccountNotFound
),
fee: 5u64,
pre_balances: pre_balances_vec.clone(),
post_balances: post_balances_vec.clone(),
inner_instructions: Some(inner_instructions_vec.clone()),
log_messages: Some(log_messages_vec.clone()),
pre_token_balances: Some(pre_token_balances_vec.clone()),
post_token_balances: Some(post_token_balances_vec.clone())
},
)
.put_protobuf((0, Signature::default(), 0), &status,)
.is_ok());
// result found
@ -5860,8 +5873,14 @@ pub mod tests {
pre_token_balances,
post_token_balances,
} = transaction_status_cf
.get((0, Signature::default(), 0))
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((
0,
Signature::default(),
0,
))
.unwrap()
.unwrap()
.try_into()
.unwrap();
assert_eq!(status, Err(TransactionError::AccountNotFound));
assert_eq!(fee, 5u64);
@ -5873,20 +5892,19 @@ pub mod tests {
assert_eq!(post_token_balances.unwrap(), post_token_balances_vec);
// insert value
let status = TransactionStatusMeta {
status: solana_sdk::transaction::Result::<()>::Ok(()),
fee: 9u64,
pre_balances: pre_balances_vec.clone(),
post_balances: post_balances_vec.clone(),
inner_instructions: Some(inner_instructions_vec.clone()),
log_messages: Some(log_messages_vec.clone()),
pre_token_balances: Some(pre_token_balances_vec.clone()),
post_token_balances: Some(post_token_balances_vec.clone()),
}
.into();
assert!(transaction_status_cf
.put(
(0, Signature::new(&[2u8; 64]), 9),
&TransactionStatusMeta {
status: solana_sdk::transaction::Result::<()>::Ok(()),
fee: 9u64,
pre_balances: pre_balances_vec.clone(),
post_balances: post_balances_vec.clone(),
inner_instructions: Some(inner_instructions_vec.clone()),
log_messages: Some(log_messages_vec.clone()),
pre_token_balances: Some(pre_token_balances_vec.clone()),
post_token_balances: Some(post_token_balances_vec.clone())
},
)
.put_protobuf((0, Signature::new(&[2u8; 64]), 9), &status,)
.is_ok());
// result found
@ -5900,8 +5918,14 @@ pub mod tests {
pre_token_balances,
post_token_balances,
} = transaction_status_cf
.get((0, Signature::new(&[2u8; 64]), 9))
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((
0,
Signature::new(&[2u8; 64]),
9,
))
.unwrap()
.unwrap()
.try_into()
.unwrap();
// deserialize
@ -5938,7 +5962,7 @@ pub mod tests {
Signature::new(&random_bytes),
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -6004,7 +6028,7 @@ pub mod tests {
Signature::new(&random_bytes),
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -6143,7 +6167,8 @@ pub mod tests {
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
};
}
.into();
let signature1 = Signature::new(&[1u8; 64]);
let signature2 = Signature::new(&[2u8; 64]);
@ -6157,46 +6182,46 @@ pub mod tests {
// signature4 in 2 non-roots,
// extra entries
transaction_status_cf
.put((0, signature2, 1), &status)
.put_protobuf((0, signature2, 1), &status)
.unwrap();
transaction_status_cf
.put((0, signature2, 2), &status)
.put_protobuf((0, signature2, 2), &status)
.unwrap();
transaction_status_cf
.put((0, signature4, 0), &status)
.put_protobuf((0, signature4, 0), &status)
.unwrap();
transaction_status_cf
.put((0, signature4, 1), &status)
.put_protobuf((0, signature4, 1), &status)
.unwrap();
transaction_status_cf
.put((0, signature5, 0), &status)
.put_protobuf((0, signature5, 0), &status)
.unwrap();
transaction_status_cf
.put((0, signature5, 1), &status)
.put_protobuf((0, signature5, 1), &status)
.unwrap();
// Initialize index 1, including:
// signature4 in non-root and root,
// extra entries
transaction_status_cf
.put((1, signature4, 1), &status)
.put_protobuf((1, signature4, 1), &status)
.unwrap();
transaction_status_cf
.put((1, signature4, 2), &status)
.put_protobuf((1, signature4, 2), &status)
.unwrap();
transaction_status_cf
.put((1, signature5, 0), &status)
.put_protobuf((1, signature5, 0), &status)
.unwrap();
transaction_status_cf
.put((1, signature5, 1), &status)
.put_protobuf((1, signature5, 1), &status)
.unwrap();
blockstore.set_roots(&[2]).unwrap();
@ -6280,21 +6305,20 @@ pub mod tests {
let pre_token_balances = Some(vec![]);
let post_token_balances = Some(vec![]);
let signature = transaction.signatures[0];
let status = TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: pre_balances.clone(),
post_balances: post_balances.clone(),
inner_instructions: inner_instructions.clone(),
log_messages: log_messages.clone(),
pre_token_balances: pre_token_balances.clone(),
post_token_balances: post_token_balances.clone(),
}
.into();
blockstore
.transaction_status_cf
.put(
(0, signature, slot),
&TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: pre_balances.clone(),
post_balances: post_balances.clone(),
inner_instructions: inner_instructions.clone(),
log_messages: log_messages.clone(),
pre_token_balances: pre_token_balances.clone(),
post_token_balances: post_token_balances.clone(),
},
)
.put_protobuf((0, signature, slot), &status)
.unwrap();
TransactionWithStatusMeta {
transaction,
@ -6366,7 +6390,7 @@ pub mod tests {
signature,
vec![&address0],
vec![&address1],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -6381,7 +6405,7 @@ pub mod tests {
signature,
vec![&address0],
vec![&address1],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -6474,7 +6498,7 @@ pub mod tests {
signature,
vec![&address0],
vec![&address1],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -6534,7 +6558,7 @@ pub mod tests {
transaction.signatures[0],
transaction.message.account_keys.iter().collect(),
vec![],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -6740,22 +6764,21 @@ pub mod tests {
vec![solana_sdk::pubkey::new_rand()],
vec![CompiledInstruction::new(1, &(), vec![0])],
);
let status = TransactionStatusMeta {
status: solana_sdk::transaction::Result::<()>::Err(
TransactionError::AccountNotFound,
),
fee: x,
pre_balances: vec![],
post_balances: vec![],
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
}
.into();
transaction_status_cf
.put(
(0, transaction.signatures[0], slot),
&TransactionStatusMeta {
status: solana_sdk::transaction::Result::<()>::Err(
TransactionError::AccountNotFound,
),
fee: x,
pre_balances: vec![],
post_balances: vec![],
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![]),
post_token_balances: Some(vec![]),
},
)
.put_protobuf((0, transaction.signatures[0], slot), &status)
.unwrap();
transactions.push(transaction);
}
@ -7264,6 +7287,73 @@ pub mod tests {
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}
#[test]
fn test_transaction_status_protobuf_backward_compatability() {
let blockstore_path = get_tmp_ledger_path!();
{
let blockstore = Blockstore::open(&blockstore_path).unwrap();
let status = TransactionStatusMeta {
status: Ok(()),
fee: 42,
pre_balances: vec![1, 2, 3],
post_balances: vec![1, 2, 3],
inner_instructions: Some(vec![]),
log_messages: Some(vec![]),
pre_token_balances: Some(vec![TransactionTokenBalance {
account_index: 0,
mint: Pubkey::new_unique().to_string(),
ui_token_amount: UiTokenAmount {
ui_amount: Some(1.1),
decimals: 1,
amount: "11".to_string(),
ui_amount_string: "1.1".to_string(),
},
}]),
post_token_balances: Some(vec![TransactionTokenBalance {
account_index: 0,
mint: Pubkey::new_unique().to_string(),
ui_token_amount: UiTokenAmount {
ui_amount: None,
decimals: 1,
amount: "11".to_string(),
ui_amount_string: "1.1".to_string(),
},
}]),
};
let deprecated_status: StoredTransactionStatusMeta = status.clone().into();
let protobuf_status: generated::TransactionStatusMeta = status.into();
for slot in 0..2 {
let data = serialize(&deprecated_status).unwrap();
blockstore
.transaction_status_cf
.put_bytes((0, Signature::default(), slot), &data)
.unwrap();
}
for slot in 2..4 {
blockstore
.transaction_status_cf
.put_protobuf((0, Signature::default(), slot), &protobuf_status)
.unwrap();
}
for slot in 0..4 {
assert_eq!(
blockstore
.transaction_status_cf
.get_protobuf_or_bincode::<StoredTransactionStatusMeta>((
0,
Signature::default(),
slot
))
.unwrap()
.unwrap(),
protobuf_status
);
}
}
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}
#[test]
fn test_remove_shred_data_complete_flag() {
let (mut shreds, entries) = make_slot_entries(0, 0, 1);

10
ledger/src/blockstore/blockstore_purge.rs
View File

@ -504,7 +504,7 @@ pub mod tests {
Signature::new(&random_bytes),
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -519,7 +519,7 @@ pub mod tests {
Signature::new(&random_bytes),
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -556,7 +556,7 @@ pub mod tests {
Signature::new(&random_bytes),
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -745,7 +745,7 @@ pub mod tests {
signature,
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}
@ -781,7 +781,7 @@ pub mod tests {
signature,
vec![&Pubkey::new(&random_bytes[0..32])],
vec![&Pubkey::new(&random_bytes[32..])],
&TransactionStatusMeta::default(),
TransactionStatusMeta::default(),
)
.unwrap();
}

8
ledger/src/blockstore_db.rs
View File

@ -17,7 +17,6 @@ use solana_sdk::{
signature::Signature,
};
use solana_storage_proto::convert::generated;
use solana_transaction_status::TransactionStatusMeta;
use std::{collections::HashMap, fs, marker::PhantomData, path::Path, sync::Arc};
use thiserror::Error;
@ -418,10 +417,6 @@ pub trait TypedColumn: Column {
type Type: Serialize + DeserializeOwned;
}
impl TypedColumn for columns::TransactionStatus {
type Type = TransactionStatusMeta;
}
impl TypedColumn for columns::AddressSignatures {
type Type = blockstore_meta::AddressSignatureMeta;
}
@ -492,6 +487,9 @@ impl Column for columns::TransactionStatus {
impl ColumnName for columns::TransactionStatus {
const NAME: &'static str = TRANSACTION_STATUS_CF;
}
impl ProtobufColumn for columns::TransactionStatus {
type Type = generated::TransactionStatusMeta;
}
impl Column for columns::AddressSignatures {
type Index = (u64, Pubkey, Slot, Signature);