gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rpc: filters performance improvement (#20185)

Browse Source 
* Add Base58,Base64,Bytes to MemcmpEncodedBytes

* Rpc: decode memcmp before filtering accounts

* Add deprecated attribute

* Add Memcmp::bytes

* Fix clippy for deprecated

* Another clippy fix

* merge RpcFilterError::DataTooLarge

* add deprecation for Base58DataTooLarge

* change filter data size limit

* strict data size len for base58

* add magic numbers

* fix tests
This commit is contained in:
Kirill Fomichev
2021-10-14 21:43:27 +03:00
committed by GitHub
parent 4beabb3a43
commit e9a427b9c8
4 changed files with 147 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
client/src/rpc_filter.rs
View File

@@ -1,4 +1,9 @@
use thiserror::Error;
#![allow(deprecated)]
use {std::borrow::Cow, thiserror::Error};
const MAX_DATA_SIZE: usize = 128;
const MAX_DATA_BASE58_SIZE: usize = 175;
const MAX_DATA_BASE64_SIZE: usize = 172;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -15,15 +20,50 @@ impl RpcFilterType {
let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
match encoding {
MemcmpEncoding::Binary => {
let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
if bytes.len() > 128 {
Err(RpcFilterError::Base58DataTooLarge)
} else {
bs58::decode(&bytes)
.into_vec()
.map(|_| ())
.map_err(|e| e.into())
use MemcmpEncodedBytes::*;
match &compare.bytes {
Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::Base58DataTooLarge)
}
Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
_ => Ok(()),
}?;
match &compare.bytes {
Binary(bytes) => {
let bytes = bs58::decode(&bytes)
.into_vec()
.map_err(RpcFilterError::DecodeError)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::Base58DataTooLarge)
} else {
Ok(())
}
}
Base58(bytes) => {
let bytes = bs58::decode(&bytes).into_vec()?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Base64(bytes) => {
let bytes = base64::decode(&bytes)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Bytes(_) => Ok(()),
}
}
}
@@ -34,10 +74,24 @@ impl RpcFilterType {
#[derive(Error, PartialEq, Debug)]
pub enum RpcFilterError {
#[error("bs58 decode error")]
DecodeError(#[from] bs58::decode::Error),
#[error("encoded binary data should be less than 129 bytes")]
DataTooLarge,
#[deprecated(
since = "1.9.0",
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
)]
#[error("encoded binary (base 58) data should be less than 129 bytes")]
Base58DataTooLarge,
#[deprecated(
since = "1.9.0",
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
)]
#[error("bs58 decode error")]
DecodeError(bs58::decode::Error),
#[error("base58 decode error")]
Base58DecodeError(#[from] bs58::decode::Error),
#[error("base64 decode error")]
Base64DecodeError(#[from] base64::DecodeError),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -49,7 +103,14 @@ pub enum MemcmpEncoding {
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", untagged)]
pub enum MemcmpEncodedBytes {
#[deprecated(
since = "1.9.0",
note = "Please use MemcmpEncodedBytes::Base58 instead"
)]
Binary(String),
Base58(String),
Base64(String),
Bytes(Vec<u8>),
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -63,14 +124,18 @@ pub struct Memcmp {
}
impl Memcmp {
pub fn bytes_match(&self, data: &[u8]) -> bool {
pub fn bytes(&self) -> Option<Cow<Vec<u8>>> {
use MemcmpEncodedBytes::*;
match &self.bytes {
MemcmpEncodedBytes::Binary(bytes) => {
let bytes = bs58::decode(bytes).into_vec();
if bytes.is_err() {
return false;
}
let bytes = bytes.unwrap();
Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned),
Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned),
Bytes(bytes) => Some(Cow::Borrowed(bytes)),
}
}
pub fn bytes_match(&self, data: &[u8]) -> bool {
match self.bytes() {
Some(bytes) => {
if self.offset > data.len() {
return false;
}
@@ -79,6 +144,7 @@ impl Memcmp {
}
data[self.offset..self.offset + bytes.len()] == bytes[..]
}
None => false,
}
}
}
@@ -87,6 +153,15 @@ impl Memcmp {
mod tests {
use super::*;
#[test]
fn test_worst_case_encoded_tx_goldens() {
let ff_data = vec![0xffu8; MAX_DATA_SIZE];
let data58 = bs58::encode(&ff_data).into_string();
assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE);
let data64 = base64::encode(&ff_data);
assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE);
}
#[test]
fn test_bytes_match() {
let data = vec![1, 2, 3, 4, 5];
@@ -94,7 +169,7 @@ mod tests {
// Exact match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
encoding: None,
}
.bytes_match(&data));
@@ -102,7 +177,7 @@ mod tests {
// Partial match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()),
encoding: None,
}
.bytes_match(&data));
@@ -110,7 +185,7 @@ mod tests {
// Offset partial match of data succeeds
assert!(Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()),
encoding: None,
}
.bytes_match(&data));
@@ -118,7 +193,7 @@ mod tests {
// Incorrect partial match of data fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()),
encoding: None,
}
.bytes_match(&data));
@@ -126,7 +201,7 @@ mod tests {
// Bytes overrun data fails
assert!(!Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()),
encoding: None,
}
.bytes_match(&data));
@@ -134,7 +209,7 @@ mod tests {
// Offset outside data fails
assert!(!Memcmp {
offset: 6,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()),
encoding: None,
}
.bytes_match(&data));
@@ -142,7 +217,7 @@ mod tests {
// Invalid base-58 fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
bytes: MemcmpEncodedBytes::Base58("III".to_string()),
encoding: None,
}
.bytes_match(&data));
@@ -157,7 +232,7 @@ mod tests {
assert_eq!(
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
encoding: None,
})
.verify(),
@@ -172,11 +247,11 @@ mod tests {
assert_eq!(
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
encoding: None,
})
.verify(),
Err(RpcFilterError::Base58DataTooLarge)
Err(RpcFilterError::DataTooLarge)
);
}
}