gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rpc: filters performance improvement (#20185) (#20862)

Browse Source 
* Add Base58,Base64,Bytes to MemcmpEncodedBytes

* Rpc: decode memcmp before filtering accounts

* Add deprecated attribute

* Add Memcmp::bytes

* Fix clippy for deprecated

* Another clippy fix

* merge RpcFilterError::DataTooLarge

* add deprecation for Base58DataTooLarge

* change filter data size limit

* strict data size len for base58

* add magic numbers

* fix tests

(cherry picked from commit e9a427b9c8)

Co-authored-by: Kirill Fomichev <fanatid@ya.ru>
This commit is contained in:
mergify[bot]
2021-10-22 01:29:49 +00:00
committed by GitHub
parent a483a7da47
commit 1d7f6a2c66
4 changed files with 147 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cli/src/cluster_query.rs
View File

@ -1721,7 +1721,7 @@ pub fn process_show_stakes(
// Filter by `StakeState::Stake(_, _)`
rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
offset: 0,
bytes: rpc_filter::MemcmpEncodedBytes::Binary(
bytes: rpc_filter::MemcmpEncodedBytes::Base58(
bs58::encode([2, 0, 0, 0]).into_string(),
),
encoding: Some(rpc_filter::MemcmpEncoding::Binary),
@ -1729,7 +1729,7 @@ pub fn process_show_stakes(
// Filter by `Delegation::voter_pubkey`, which begins at byte offset 124
rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
offset: 124,
bytes: rpc_filter::MemcmpEncodedBytes::Binary(
bytes: rpc_filter::MemcmpEncodedBytes::Base58(
vote_account_pubkeys[0].to_string(),
),
encoding: Some(rpc_filter::MemcmpEncoding::Binary),

14
cli/src/program.rs
View File

@ -1146,18 +1146,18 @@ fn get_buffers(
) -> Result<CliUpgradeableBuffers, Box<dyn std::error::Error>> {
let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 0, 0, 0]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 0, 0, 0]).into_string()),
encoding: None,
})];
if let Some(authority_pubkey) = authority_pubkey {
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
encoding: None,
}));
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE + OPTION_SIZE,
bytes: MemcmpEncodedBytes::Binary(
bytes: MemcmpEncodedBytes::Base58(
bs58::encode(authority_pubkey.as_ref()).into_string(),
),
encoding: None,
@ -1199,18 +1199,18 @@ fn get_programs(
) -> Result<CliUpgradeablePrograms, Box<dyn std::error::Error>> {
let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 0, 0, 0]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 0, 0, 0]).into_string()),
encoding: None,
})];
if let Some(authority_pubkey) = authority_pubkey {
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
encoding: None,
}));
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE + OPTION_SIZE,
bytes: MemcmpEncodedBytes::Binary(
bytes: MemcmpEncodedBytes::Base58(
bs58::encode(authority_pubkey.as_ref()).into_string(),
),
encoding: None,
@ -1234,7 +1234,7 @@ fn get_programs(
bytes.extend_from_slice(programdata_address.as_ref());
let filters = vec![RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(bytes).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(bytes).into_string()),
encoding: None,
})];

133
client/src/rpc_filter.rs
View File

@ -1,4 +1,9 @@
use thiserror::Error;
#![allow(deprecated)]
use {std::borrow::Cow, thiserror::Error};
const MAX_DATA_SIZE: usize = 128;
const MAX_DATA_BASE58_SIZE: usize = 175;
const MAX_DATA_BASE64_SIZE: usize = 172;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -15,15 +20,50 @@ impl RpcFilterType {
let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
match encoding {
MemcmpEncoding::Binary => {
let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
if bytes.len() > 128 {
Err(RpcFilterError::Base58DataTooLarge)
} else {
bs58::decode(&bytes)
.into_vec()
.map(|_| ())
.map_err(|e| e.into())
use MemcmpEncodedBytes::*;
match &compare.bytes {
Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::Base58DataTooLarge)
}
Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
_ => Ok(()),
}?;
match &compare.bytes {
Binary(bytes) => {
let bytes = bs58::decode(&bytes)
.into_vec()
.map_err(RpcFilterError::DecodeError)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::Base58DataTooLarge)
} else {
Ok(())
}
}
Base58(bytes) => {
let bytes = bs58::decode(&bytes).into_vec()?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Base64(bytes) => {
let bytes = base64::decode(&bytes)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Bytes(_) => Ok(()),
}
}
}
@ -34,10 +74,24 @@ impl RpcFilterType {
#[derive(Error, PartialEq, Debug)]
pub enum RpcFilterError {
#[error("bs58 decode error")]
DecodeError(#[from] bs58::decode::Error),
#[error("encoded binary data should be less than 129 bytes")]
DataTooLarge,
#[deprecated(
since = "1.9.0",
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
)]
#[error("encoded binary (base 58) data should be less than 129 bytes")]
Base58DataTooLarge,
#[deprecated(
since = "1.9.0",
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
)]
#[error("bs58 decode error")]
DecodeError(bs58::decode::Error),
#[error("base58 decode error")]
Base58DecodeError(#[from] bs58::decode::Error),
#[error("base64 decode error")]
Base64DecodeError(#[from] base64::DecodeError),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@ -49,7 +103,14 @@ pub enum MemcmpEncoding {
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", untagged)]
pub enum MemcmpEncodedBytes {
#[deprecated(
since = "1.9.0",
note = "Please use MemcmpEncodedBytes::Base58 instead"
)]
Binary(String),
Base58(String),
Base64(String),
Bytes(Vec<u8>),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@ -63,14 +124,18 @@ pub struct Memcmp {
}
impl Memcmp {
pub fn bytes_match(&self, data: &[u8]) -> bool {
pub fn bytes(&self) -> Option<Cow<Vec<u8>>> {
use MemcmpEncodedBytes::*;
match &self.bytes {
MemcmpEncodedBytes::Binary(bytes) => {
let bytes = bs58::decode(bytes).into_vec();
if bytes.is_err() {
return false;
}
let bytes = bytes.unwrap();
Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned),
Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned),
Bytes(bytes) => Some(Cow::Borrowed(bytes)),
}
}
pub fn bytes_match(&self, data: &[u8]) -> bool {
match self.bytes() {
Some(bytes) => {
if self.offset > data.len() {
return false;
}
@ -79,6 +144,7 @@ impl Memcmp {
}
data[self.offset..self.offset + bytes.len()] == bytes[..]
}
None => false,
}
}
}
@ -87,6 +153,15 @@ impl Memcmp {
mod tests {
use super::*;
#[test]
fn test_worst_case_encoded_tx_goldens() {
let ff_data = vec![0xffu8; MAX_DATA_SIZE];
let data58 = bs58::encode(&ff_data).into_string();
assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE);
let data64 = base64::encode(&ff_data);
assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE);
}
#[test]
fn test_bytes_match() {
let data = vec![1, 2, 3, 4, 5];
@ -94,7 +169,7 @@ mod tests {
// Exact match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
encoding: None,
}
.bytes_match(&data));
@ -102,7 +177,7 @@ mod tests {
// Partial match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()),
encoding: None,
}
.bytes_match(&data));
@ -110,7 +185,7 @@ mod tests {
// Offset partial match of data succeeds
assert!(Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()),
encoding: None,
}
.bytes_match(&data));
@ -118,7 +193,7 @@ mod tests {
// Incorrect partial match of data fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()),
encoding: None,
}
.bytes_match(&data));
@ -126,7 +201,7 @@ mod tests {
// Bytes overrun data fails
assert!(!Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()),
encoding: None,
}
.bytes_match(&data));
@ -134,7 +209,7 @@ mod tests {
// Offset outside data fails
assert!(!Memcmp {
offset: 6,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()),
encoding: None,
}
.bytes_match(&data));
@ -142,7 +217,7 @@ mod tests {
// Invalid base-58 fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
bytes: MemcmpEncodedBytes::Base58("III".to_string()),
encoding: None,
}
.bytes_match(&data));
@ -157,7 +232,7 @@ mod tests {
assert_eq!(
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
encoding: None,
})
.verify(),
@ -172,11 +247,11 @@ mod tests {
assert_eq!(
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
encoding: None,
})
.verify(),
Err(RpcFilterError::Base58DataTooLarge)
Err(RpcFilterError::DataTooLarge)
);
}
}

49
rpc/src/rpc.rs
View File

@ -1688,7 +1688,7 @@ impl JsonRpcRequestProcessor {
// Optional filter on Mint address
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(mint.to_string()),
bytes: MemcmpEncodedBytes::Bytes(mint.to_bytes().into()),
encoding: None,
}));
}
@ -1732,15 +1732,13 @@ impl JsonRpcRequestProcessor {
// Filter on Delegate is_some()
RpcFilterType::Memcmp(Memcmp {
offset: 72,
bytes: MemcmpEncodedBytes::Binary(
bs58::encode(bincode::serialize(&1u32).unwrap()).into_string(),
),
bytes: MemcmpEncodedBytes::Bytes(bincode::serialize(&1u32).unwrap()),
encoding: None,
}),
// Filter on Delegate address
RpcFilterType::Memcmp(Memcmp {
offset: 76,
bytes: MemcmpEncodedBytes::Binary(delegate.to_string()),
bytes: MemcmpEncodedBytes::Bytes(delegate.to_bytes().into()),
encoding: None,
}),
];
@ -1779,8 +1777,9 @@ impl JsonRpcRequestProcessor {
&self,
bank: &Arc<Bank>,
program_id: &Pubkey,
filters: Vec<RpcFilterType>,
mut filters: Vec<RpcFilterType>,
) -> RpcCustomResult<Vec<(Pubkey, AccountSharedData)>> {
optimize_filters(&mut filters);
let filter_closure = |account: &AccountSharedData| {
filters.iter().all(|filter_type| match filter_type {
RpcFilterType::DataSize(size) => account.data().len() as u64 == *size,
@ -1837,7 +1836,7 @@ impl JsonRpcRequestProcessor {
// Filter on Owner address
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET,
bytes: MemcmpEncodedBytes::Binary(owner_key.to_string()),
bytes: MemcmpEncodedBytes::Bytes(owner_key.to_bytes().into()),
encoding: None,
}));
@ -1851,6 +1850,7 @@ impl JsonRpcRequestProcessor {
index_key: owner_key.to_string(),
});
}
optimize_filters(&mut filters);
Ok(bank
.get_filtered_indexed_accounts(&IndexKey::SplTokenOwner(*owner_key), |account| {
account.owner() == &spl_token_id_v2_0()
@ -1886,7 +1886,7 @@ impl JsonRpcRequestProcessor {
// Filter on Mint address
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: SPL_TOKEN_ACCOUNT_MINT_OFFSET,
bytes: MemcmpEncodedBytes::Binary(mint_key.to_string()),
bytes: MemcmpEncodedBytes::Bytes(mint_key.to_bytes().into()),
encoding: None,
}));
if self
@ -1899,6 +1899,7 @@ impl JsonRpcRequestProcessor {
index_key: mint_key.to_string(),
});
}
optimize_filters(&mut filters);
Ok(bank
.get_filtered_indexed_accounts(&IndexKey::SplTokenMint(*mint_key), |account| {
account.owner() == &spl_token_id_v2_0()
@ -1916,6 +1917,24 @@ impl JsonRpcRequestProcessor {
}
}
fn optimize_filters(filters: &mut Vec<RpcFilterType>) {
filters.iter_mut().for_each(|filter_type| {
if let RpcFilterType::Memcmp(compare) = filter_type {
use MemcmpEncodedBytes::*;
match &compare.bytes {
#[allow(deprecated)]
Binary(bytes) | Base58(bytes) => {
compare.bytes = Bytes(bs58::decode(bytes).into_vec().unwrap());
}
Base64(bytes) => {
compare.bytes = Bytes(base64::decode(bytes).unwrap());
}
_ => {}
}
}
})
}
fn verify_transaction(
transaction: &Transaction,
libsecp256k1_0_5_upgrade_enabled: bool,
@ -2082,7 +2101,7 @@ fn get_spl_token_owner_filter(program_id: &Pubkey, filters: &[RpcFilterType]) ->
RpcFilterType::DataSize(size) => data_size_filter = Some(*size),
RpcFilterType::Memcmp(Memcmp {
offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET,
bytes: MemcmpEncodedBytes::Binary(bytes),
bytes: MemcmpEncodedBytes::Base58(bytes),
..
}) => {
if let Ok(key) = Pubkey::from_str(bytes) {
@ -2110,7 +2129,7 @@ fn get_spl_token_mint_filter(program_id: &Pubkey, filters: &[RpcFilterType]) ->
RpcFilterType::DataSize(size) => data_size_filter = Some(*size),
RpcFilterType::Memcmp(Memcmp {
offset: SPL_TOKEN_ACCOUNT_MINT_OFFSET,
bytes: MemcmpEncodedBytes::Binary(bytes),
bytes: MemcmpEncodedBytes::Base58(bytes),
..
}) => {
if let Ok(key) = Pubkey::from_str(bytes) {
@ -5895,7 +5914,7 @@ pub mod tests {
fn test_rpc_verify_filter() {
let filter = RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(
bytes: MemcmpEncodedBytes::Base58(
"13LeFbG6m2EP1fqCj9k66fcXsoTHMMtgr7c78AivUrYD".to_string(),
),
encoding: None,
@ -5904,7 +5923,7 @@ pub mod tests {
// Invalid base-58
let filter = RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
bytes: MemcmpEncodedBytes::Base58("III".to_string()),
encoding: None,
});
assert!(verify_filter(&filter).is_err());
@ -7400,7 +7419,7 @@ pub mod tests {
&[
RpcFilterType::Memcmp(Memcmp {
offset: 32,
bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
bytes: MemcmpEncodedBytes::Base58(owner.to_string()),
encoding: None
}),
RpcFilterType::DataSize(165)
@ -7416,7 +7435,7 @@ pub mod tests {
&[
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
bytes: MemcmpEncodedBytes::Base58(owner.to_string()),
encoding: None
}),
RpcFilterType::DataSize(165)
@ -7430,7 +7449,7 @@ pub mod tests {
&[
RpcFilterType::Memcmp(Memcmp {
offset: 32,
bytes: MemcmpEncodedBytes::Binary(owner.to_string()),
bytes: MemcmpEncodedBytes::Base58(owner.to_string()),
encoding: None
}),
RpcFilterType::DataSize(165)