sdk: Add Borsh support for types and utilities (#15290)

* sdk: Add Borsh to Pubkey

* Add serialization error for easier borsh integration

* Add Borsh usage to banks-client and sdk

* Rename SerializationError -> IOError

* Add new errors to proto

* Update Cargo lock

* Update Cargo.lock based on CI

* Clippy

* Update ABI on bank

* Address review feedback

* Update sanity program instruction count test

Cargo.lock

@@ -350,6 +350,51 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
 
+[[package]]
+name = "borsh"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5a26c53ddf60281f18e7a29b20db7ba3db82a9d81b9650bfaa02d646f50d364"
+dependencies = [
+ "borsh-derive",
+ "hashbrown 0.9.1",
+]
+
+[[package]]
+name = "borsh-derive"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b637a47728b78a78cd7f4b85bf06d71ef4221840e059a38f048be2422bf673b2"
+dependencies = [
+ "borsh-derive-internal",
+ "borsh-schema-derive-internal",
+ "proc-macro-crate",
+ "proc-macro2 1.0.24",
+ "syn 1.0.60",
+]
+
+[[package]]
+name = "borsh-derive-internal"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d813fa25eb0bed78c36492cff4415f38c760d6de833d255ba9095bd8ebb7d725"
+dependencies = [
+ "proc-macro2 1.0.24",
+ "quote 1.0.6",
+ "syn 1.0.60",
+]
+
+[[package]]
+name = "borsh-schema-derive-internal"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf78ee4a98c8cb9eba1bac3d3e2a1ea3d7673c719ce691e67b5cbafc472d3b7"
+dependencies = [
+ "proc-macro2 1.0.24",
+ "quote 1.0.6",
+ "syn 1.0.60",
+]
+
 [[package]]
 name = "bs58"
 version = "0.3.1"
@@ -2915,9 +2960,9 @@ checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131"
 
 [[package]]
 name = "proc-macro-crate"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e"
+checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
 dependencies = [
  "toml",
 ]
@@ -3942,10 +3987,13 @@ name = "solana-banks-client"
 version = "1.6.0"
 dependencies = [
  "bincode",
+ "borsh",
+ "borsh-derive",
  "futures 0.3.8",
  "mio 0.7.6",
  "solana-banks-interface",
  "solana-banks-server",
+ "solana-program 1.6.0",
  "solana-runtime",
  "solana-sdk",
  "tarpc",
@@ -4904,6 +4952,8 @@ version = "1.6.0"
 dependencies = [
  "assert_matches",
  "bincode",
+ "borsh",
+ "borsh-derive",
  "bs58",
  "bv",
  "curve25519-dalek 2.1.0",

banks-client/Cargo.toml

@@ -10,9 +10,12 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.3.1"
+borsh = "0.8.1"
+borsh-derive = "0.8.1"
 futures = "0.3"
 mio = "0.7.6"
 solana-banks-interface = { path = "../banks-interface", version = "1.6.0" }
+solana-program = { path = "../sdk/program", version = "1.6.0" }
 solana-sdk = { path = "../sdk", version = "1.6.0" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1.1", features = ["full"] }

banks-client/src/lib.rs

@@ -5,19 +5,18 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
+use borsh::BorshDeserialize;
 use futures::{future::join_all, Future, FutureExt};
 pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
 use solana_banks_interface::{BanksRequest, BanksResponse};
+use solana_program::{
+    clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
+    rent::Rent, sysvar,
+};
 use solana_sdk::{
     account::{from_account, Account},
-    clock::Slot,
     commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    pubkey::Pubkey,
-    rent::Rent,
     signature::Signature,
-    sysvar,
     transaction::{self, Transaction},
     transport,
 };
@@ -218,6 +217,33 @@ impl BanksClient {
         self.get_account_with_commitment(address, CommitmentLevel::default())
     }
 
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_packed_account_data<T: Pack>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account =
+                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Account not found"))?;
+            T::unpack_from_slice(&account.data)
+                .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to deserialize account"))
+        })
+    }
+
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_account_data_with_borsh<T: BorshDeserialize>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account =
+                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))?;
+            T::try_from_slice(&account.data)
+        })
+    }
+
     /// Return the balance in lamports of an account at the given address at the slot
     /// corresponding to the given commitment level.
     pub fn get_balance_with_commitment(

program-test/src/lib.rs

@@ -72,6 +72,8 @@ pub fn to_instruction_error(error: ProgramError) -> InstructionError {
         ProgramError::AccountBorrowFailed => InstructionError::AccountBorrowFailed,
         ProgramError::MaxSeedLengthExceeded => InstructionError::MaxSeedLengthExceeded,
         ProgramError::InvalidSeeds => InstructionError::InvalidSeeds,
+        ProgramError::IOError(err) => InstructionError::IOError(err),
+        ProgramError::AccountNotRentExempt => InstructionError::AccountNotRentExempt,
     }
 }
 

programs/bpf/Cargo.lock

@@ -25,6 +25,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d"
 
+[[package]]
+name = "ahash"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.10"
@@ -196,6 +202,51 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
 
+[[package]]
+name = "borsh"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5a26c53ddf60281f18e7a29b20db7ba3db82a9d81b9650bfaa02d646f50d364"
+dependencies = [
+ "borsh-derive",
+ "hashbrown",
+]
+
+[[package]]
+name = "borsh-derive"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b637a47728b78a78cd7f4b85bf06d71ef4221840e059a38f048be2422bf673b2"
+dependencies = [
+ "borsh-derive-internal",
+ "borsh-schema-derive-internal",
+ "proc-macro-crate",
+ "proc-macro2 1.0.24",
+ "syn 1.0.60",
+]
+
+[[package]]
+name = "borsh-derive-internal"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d813fa25eb0bed78c36492cff4415f38c760d6de833d255ba9095bd8ebb7d725"
+dependencies = [
+ "proc-macro2 1.0.24",
+ "quote 1.0.6",
+ "syn 1.0.60",
+]
+
+[[package]]
+name = "borsh-schema-derive-internal"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf78ee4a98c8cb9eba1bac3d3e2a1ea3d7673c719ce691e67b5cbafc472d3b7"
+dependencies = [
+ "proc-macro2 1.0.24",
+ "quote 1.0.6",
+ "syn 1.0.60",
+]
+
 [[package]]
 name = "bs58"
 version = "0.3.1"
@@ -1011,6 +1062,15 @@ dependencies = [
  "byteorder 1.3.4",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+dependencies = [
+ "ahash",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.13"
@@ -3079,6 +3139,8 @@ name = "solana-program"
 version = "1.6.0"
 dependencies = [
  "bincode",
+ "borsh",
+ "borsh-derive",
  "bs58",
  "bv",
  "curve25519-dalek 2.1.0",

programs/bpf/tests/programs.rs

@@ -1233,7 +1233,7 @@ fn assert_instruction_count() {
             ("solana_bpf_rust_noop", 488),
             ("solana_bpf_rust_param_passing", 48),
             ("solana_bpf_rust_ristretto", 19399),
-            ("solana_bpf_rust_sanity", 894),
+            ("solana_bpf_rust_sanity", 935),
         ]);
     }
 

runtime/src/bank.rs

@@ -109,7 +109,7 @@ impl ExecuteTimings {
 }
 
 type BankStatusCache = StatusCache<Result<()>>;
-#[frozen_abi(digest = "MUmkgPsCRrWL2HEsMEvpkWMis35kbBnaEZtrph5P6bk")]
+#[frozen_abi(digest = "GESbL4xC5ndicqqZeCW4fEHtNR71NMESVWM3i2Ty2MY9")]
 pub type BankSlotDelta = SlotDelta<Result<()>>;
 type TransactionAccountRefCells = Vec<Rc<RefCell<Account>>>;
 type TransactionAccountDepRefCells = Vec<(Pubkey, RefCell<Account>)>;

sdk/program/Cargo.toml

@@ -10,6 +10,8 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.3.1"
+borsh = "0.8.1"
+borsh-derive = "0.8.1"
 bs58 = "0.3.1"
 bv = { version = "0.11.1", features = ["serde"] }
 hex = "0.4.2"

sdk/program/src/borsh.rs

@@ -0,0 +1,55 @@
+//! Borsh utils
+use borsh::schema::{BorshSchema, Declaration, Definition, Fields};
+use std::collections::HashMap;
+
+/// Get packed length for the given BorchSchema Declaration
+fn get_declaration_packed_len(
+    declaration: &str,
+    definitions: &HashMap<Declaration, Definition>,
+) -> usize {
+    match definitions.get(declaration) {
+        Some(Definition::Array { length, elements }) => {
+            *length as usize * get_declaration_packed_len(elements, definitions)
+        }
+        Some(Definition::Enum { variants }) => {
+            1 + variants
+                .iter()
+                .map(|(_, declaration)| get_declaration_packed_len(declaration, definitions))
+                .max()
+                .unwrap_or(0)
+        }
+        Some(Definition::Struct { fields }) => match fields {
+            Fields::NamedFields(named_fields) => named_fields
+                .iter()
+                .map(|(_, declaration)| get_declaration_packed_len(declaration, definitions))
+                .sum(),
+            Fields::UnnamedFields(declarations) => declarations
+                .iter()
+                .map(|declaration| get_declaration_packed_len(declaration, definitions))
+                .sum(),
+            Fields::Empty => 0,
+        },
+        Some(Definition::Sequence {
+            elements: _elements,
+        }) => panic!("Missing support for Definition::Sequence"),
+        Some(Definition::Tuple { elements }) => elements
+            .iter()
+            .map(|element| get_declaration_packed_len(element, definitions))
+            .sum(),
+        None => match declaration {
+            "u8" | "i8" => 1,
+            "u16" | "i16" => 2,
+            "u32" | "i32" => 2,
+            "u64" | "i64" => 8,
+            "u128" | "i128" => 16,
+            "nil" => 0,
+            _ => panic!("Missing primitive type: {}", declaration),
+        },
+    }
+}
+
+/// Get the worst-case packed length for the given BorshSchema
+pub fn get_packed_len<S: BorshSchema>() -> usize {
+    let schema_container = S::schema_container();
+    get_declaration_packed_len(&schema_container.declaration, &schema_container.definitions)
+}

sdk/program/src/hash.rs

@@ -1,6 +1,7 @@
 //! The `hash` module provides functions for creating SHA-256 hashes.
 
 use crate::sanitize::Sanitize;
+use borsh::{BorshDeserialize, BorshSchema, BorshSerialize};
 use sha2::{Digest, Sha256};
 use std::{convert::TryFrom, fmt, mem, str::FromStr};
 use thiserror::Error;
@@ -9,7 +10,20 @@ pub const HASH_BYTES: usize = 32;
 /// Maximum string length of a base58 encoded hash
 const MAX_BASE58_LEN: usize = 44;
 #[derive(
-    Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash, AbiExample,
+    Serialize,
+    Deserialize,
+    BorshSerialize,
+    BorshDeserialize,
+    BorshSchema,
+    Clone,
+    Copy,
+    Default,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    AbiExample,
 )]
 #[repr(transparent)]
 pub struct Hash(pub [u8; HASH_BYTES]);

sdk/program/src/instruction.rs

@@ -3,6 +3,7 @@
 use crate::sanitize::Sanitize;
 use crate::{pubkey::Pubkey, short_vec};
 use bincode::serialize;
+use borsh::BorshSerialize;
 use serde::Serialize;
 use thiserror::Error;
 
@@ -186,6 +187,12 @@ pub enum InstructionError {
 
     #[error("Incorrect authority provided")]
     IncorrectAuthority,
+
+    #[error("Failed to serialize or deserialize account data: {0}")]
+    IOError(String),
+
+    #[error("An account does not have enough lamports to be rent-exempt")]
+    AccountNotRentExempt,
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
@@ -207,6 +214,19 @@ impl Instruction {
             accounts,
         }
     }
+
+    pub fn new_with_borsh<T: BorshSerialize>(
+        program_id: Pubkey,
+        data: &T,
+        accounts: Vec<AccountMeta>,
+    ) -> Self {
+        let data = data.try_to_vec().unwrap();
+        Self {
+            program_id,
+            data,
+            accounts,
+        }
+    }
 }
 
 pub fn checked_add(a: u64, b: u64) -> Result<u64, InstructionError> {

sdk/program/src/lib.rs

@@ -6,6 +6,7 @@
 extern crate self as solana_program;
 
 pub mod account_info;
+pub mod borsh;
 pub mod bpf_loader;
 pub mod bpf_loader_deprecated;
 pub mod bpf_loader_upgradeable;

sdk/program/src/program_error.rs

@@ -1,4 +1,5 @@
 use crate::{decode_error::DecodeError, instruction::InstructionError, msg, pubkey::PubkeyError};
+use borsh::maybestd::io::Error as IOError;
 use num_traits::{FromPrimitive, ToPrimitive};
 use std::convert::TryFrom;
 use thiserror::Error;
@@ -37,6 +38,10 @@ pub enum ProgramError {
     MaxSeedLengthExceeded,
     #[error("Provided seeds do not result in a valid address")]
     InvalidSeeds,
+    #[error("IO Error: {0}")]
+    IOError(String),
+    #[error("An account does not have enough lamports to be rent-exempt")]
+    AccountNotRentExempt,
 }
 
 pub trait PrintProgramError {
@@ -71,6 +76,8 @@ impl PrintProgramError for ProgramError {
             Self::AccountBorrowFailed => msg!("Error: AccountBorrowFailed"),
             Self::MaxSeedLengthExceeded => msg!("Error: MaxSeedLengthExceeded"),
             Self::InvalidSeeds => msg!("Error: InvalidSeeds"),
+            Self::IOError(_) => msg!("Error: IOError"),
+            Self::AccountNotRentExempt => msg!("Error: AccountNotRentExempt"),
         }
     }
 }
@@ -97,6 +104,8 @@ pub const NOT_ENOUGH_ACCOUNT_KEYS: u64 = to_builtin!(11);
 pub const ACCOUNT_BORROW_FAILED: u64 = to_builtin!(12);
 pub const MAX_SEED_LENGTH_EXCEEDED: u64 = to_builtin!(13);
 pub const INVALID_SEEDS: u64 = to_builtin!(14);
+pub const IO_ERROR: u64 = to_builtin!(15);
+pub const ACCOUNT_NOT_RENT_EXEMPT: u64 = to_builtin!(16);
 
 impl From<ProgramError> for u64 {
     fn from(error: ProgramError) -> Self {
@@ -114,6 +123,8 @@ impl From<ProgramError> for u64 {
             ProgramError::AccountBorrowFailed => ACCOUNT_BORROW_FAILED,
             ProgramError::MaxSeedLengthExceeded => MAX_SEED_LENGTH_EXCEEDED,
             ProgramError::InvalidSeeds => INVALID_SEEDS,
+            ProgramError::IOError(_) => IO_ERROR,
+            ProgramError::AccountNotRentExempt => ACCOUNT_NOT_RENT_EXEMPT,
 
             ProgramError::Custom(error) => {
                 if error == 0 {
@@ -166,6 +177,8 @@ impl TryFrom<InstructionError> for ProgramError {
             Self::Error::NotEnoughAccountKeys => Ok(Self::NotEnoughAccountKeys),
             Self::Error::AccountBorrowFailed => Ok(Self::AccountBorrowFailed),
             Self::Error::MaxSeedLengthExceeded => Ok(Self::MaxSeedLengthExceeded),
+            Self::Error::IOError(err) => Ok(Self::IOError(err)),
+            Self::Error::AccountNotRentExempt => Ok(Self::AccountNotRentExempt),
             _ => Err(error),
         }
     }
@@ -212,3 +225,9 @@ impl From<PubkeyError> for ProgramError {
         }
     }
 }
+
+impl From<IOError> for ProgramError {
+    fn from(error: IOError) -> Self {
+        ProgramError::IOError(format!("{}", error))
+    }
+}

sdk/program/src/pubkey.rs

@@ -1,4 +1,5 @@
 use crate::{decode_error::DecodeError, hash::hashv};
+use borsh::{BorshDeserialize, BorshSchema, BorshSerialize};
 use num_derive::{FromPrimitive, ToPrimitive};
 use std::{convert::TryFrom, fmt, mem, str::FromStr};
 use thiserror::Error;
@@ -37,7 +38,20 @@ impl From<u64> for PubkeyError {
 
 #[repr(transparent)]
 #[derive(
-    Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash, AbiExample,
+    Serialize,
+    Deserialize,
+    BorshSerialize,
+    BorshDeserialize,
+    BorshSchema,
+    Clone,
+    Copy,
+    Default,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    AbiExample,
 )]
 pub struct Pubkey([u8; 32]);
 

storage-proto/proto/solana.storage.transaction_by_addr.rs

@@ -114,4 +114,6 @@ pub enum InstructionErrorType {
     ProgramFailedToCompile = 41,
     Immutable = 42,
     IncorrectAuthority = 43,
+    IOError = 44,
+    AccountNotRentExempt = 45,
 }

storage-proto/src/convert.rs

@@ -700,6 +700,12 @@ impl From<TransactionError> for tx_by_addr::TransactionError {
                             InstructionError::IncorrectAuthority => {
                                 tx_by_addr::InstructionErrorType::IncorrectAuthority
                             }
+                            InstructionError::IOError(_) => {
+                                tx_by_addr::InstructionErrorType::IOError
+                            }
+                            InstructionError::AccountNotRentExempt => {
+                                tx_by_addr::InstructionErrorType::AccountNotRentExempt
+                            }
                         } as i32,
                         custom: match instruction_error {
                             InstructionError::Custom(custom) => {