Add get_processed_sibling_instruction syscall (#22859) (#22956)

program-runtime/src/instruction_recorder.rs

@@ -7,9 +7,9 @@ use {
 };
 
 /// Records and compiles cross-program invoked instructions
-#[derive(Clone, Default)]
+#[derive(Clone, Debug, Default, PartialEq)]
 pub struct InstructionRecorder {
-    inner: Rc<RefCell<Vec<Instruction>>>,
+    inner: Rc<RefCell<Vec<(usize, Instruction)>>>,
 }
 
 impl InstructionRecorder {
@@ -20,11 +20,39 @@ impl InstructionRecorder {
         self.inner
             .borrow()
             .iter()
-            .map(|ix| message.try_compile_instruction(ix))
+            .skip(1)
+            .map(|(_, ix)| message.try_compile_instruction(ix))
             .collect()
     }
 
-    pub fn record_instruction(&self, instruction: Instruction) {
+    pub fn record_instruction(&self, stack_height: usize, instruction: Instruction) {
-        self.inner.borrow_mut().push(instruction);
+        self.inner.borrow_mut().push((stack_height, instruction));
+    }
+
+    pub fn get(&self, index: usize) -> Option<Instruction> {
+        self.inner
+            .borrow()
+            .get(index)
+            .map(|(_, instruction)| instruction.clone())
+    }
+
+    pub fn find(&self, stack_height: usize, index: usize) -> Option<Instruction> {
+        let mut current_index = 0;
+        self.inner
+            .borrow()
+            .iter()
+            .rev()
+            .skip(1)
+            .find(|(this_stack_height, _)| {
+                if stack_height == *this_stack_height {
+                    if index == current_index {
+                        return true;
+                    } else {
+                        current_index = current_index.saturating_add(1);
+                    }
+                }
+                false
+            })
+            .map(|(_, instruction)| instruction.clone())
     }
 }

program-runtime/src/invoke_context.rs

@@ -21,7 +21,10 @@ use {
             tx_wide_compute_cap, FeatureSet,
         },
         hash::Hash,
-        instruction::{AccountMeta, CompiledInstruction, Instruction, InstructionError},
+        instruction::{
+            AccountMeta, CompiledInstruction, Instruction, InstructionError,
+            TRANSACTION_LEVEL_STACK_HEIGHT,
+        },
         keyed_account::{create_keyed_accounts_unified, keyed_account_at_index, KeyedAccount},
         message::{Message, SanitizedMessage},
         pubkey::Pubkey,
@@ -201,7 +204,7 @@ pub struct InvokeContext<'a> {
     compute_meter: Rc<RefCell<ComputeMeter>>,
     accounts_data_meter: AccountsDataMeter,
     executors: Rc<RefCell<Executors>>,
-    pub instruction_recorder: Option<&'a InstructionRecorder>,
+    pub instruction_trace: Vec<InstructionRecorder>,
     pub feature_set: Arc<FeatureSet>,
     pub timings: ExecuteDetailsTimings,
     pub blockhash: Hash,
@@ -237,7 +240,7 @@ impl<'a> InvokeContext<'a> {
             compute_meter: ComputeMeter::new_ref(compute_budget.max_units),
             accounts_data_meter: AccountsDataMeter::new(current_accounts_data_len),
             executors,
-            instruction_recorder: None,
+            instruction_trace: Vec::new(),
             feature_set,
             timings: ExecuteDetailsTimings::default(),
             blockhash,
@@ -375,8 +378,8 @@ impl<'a> InvokeContext<'a> {
         self.invoke_stack.pop();
     }
 
-    /// Current depth of the invocation stack
+    /// Current height of the stack
-    pub fn invoke_depth(&self) -> usize {
+    pub fn get_stack_height(&self) -> usize {
         self.invoke_stack.len()
     }
 
@@ -566,9 +569,7 @@ impl<'a> InvokeContext<'a> {
             prev_account_sizes.push((account, account_length));
         }
 
-        if let Some(instruction_recorder) = &self.instruction_recorder {
+        self.record_instruction(self.get_stack_height(), instruction);
-            instruction_recorder.record_instruction(instruction);
-        }
 
         let message = SanitizedMessage::Legacy(message);
         self.process_instruction(
@@ -954,6 +955,29 @@ impl<'a> InvokeContext<'a> {
     pub fn get_sysvar_cache(&self) -> &SysvarCache {
         &self.sysvar_cache
     }
+
+    /// Record top-level instruction in the instruction trace
+    pub fn record_top_level_instruction(&mut self, instruction: Instruction) {
+        self.instruction_trace.push(InstructionRecorder::default());
+        self.record_instruction(TRANSACTION_LEVEL_STACK_HEIGHT, instruction);
+    }
+
+    /// Record instruction in the instruction trace
+    pub fn record_instruction(&mut self, stack_height: usize, instruction: Instruction) {
+        if let Some(instruction_recorder) = self.instruction_trace.last() {
+            instruction_recorder.record_instruction(stack_height, instruction)
+        }
+    }
+
+    /// Get the instruction trace
+    pub fn get_instruction_trace(&self) -> &[InstructionRecorder] {
+        &self.instruction_trace
+    }
+
+    /// Get the mutable instruction trace
+    pub fn get_instruction_trace_mut(&mut self) -> &mut Vec<InstructionRecorder> {
+        &mut self.instruction_trace
+    }
 }
 
 pub struct MockInvokeContextPreparation {

program-test/src/lib.rs

@@ -98,7 +98,11 @@ pub fn builtin_process_instruction(
 
     let log_collector = invoke_context.get_log_collector();
     let program_id = invoke_context.get_caller()?;
-    stable_log::program_invoke(&log_collector, program_id, invoke_context.invoke_depth());
+    stable_log::program_invoke(
+        &log_collector,
+        program_id,
+        invoke_context.get_stack_height(),
+    );
 
     // Skip the processor account
     let keyed_accounts = &invoke_context.get_keyed_accounts()?[1..];
@@ -246,7 +250,11 @@ impl solana_sdk::program_stubs::SyscallStubs for SyscallStubs {
             .map(|(i, _)| message.is_writable(i))
             .collect::<Vec<bool>>();
 
-        stable_log::program_invoke(&log_collector, &program_id, invoke_context.invoke_depth());
+        stable_log::program_invoke(
+            &log_collector,
+            &program_id,
+            invoke_context.get_stack_height(),
+        );
 
         // Convert AccountInfos into Accounts
         let mut account_indices = Vec::with_capacity(message.account_keys.len());
@@ -305,9 +313,7 @@ impl solana_sdk::program_stubs::SyscallStubs for SyscallStubs {
             }
         }
 
-        if let Some(instruction_recorder) = &invoke_context.instruction_recorder {
+        invoke_context.record_instruction(invoke_context.get_stack_height(), instruction.clone());
-            instruction_recorder.record_instruction(instruction.clone());
-        }
 
         let message = SanitizedMessage::Legacy(message);
         invoke_context

programs/bpf/Cargo.lock

@@ -2772,6 +2772,20 @@ dependencies = [
  "solana-program 1.9.6",
 ]
 
+[[package]]
+name = "solana-bpf-rust-sibling-instructions"
+version = "1.9.6"
+dependencies = [
+ "solana-program 1.9.6",
+]
+
+[[package]]
+name = "solana-bpf-rust-sibling_inner-instructions"
+version = "1.9.6"
+dependencies = [
+ "solana-program 1.9.6",
+]
+
 [[package]]
 name = "solana-bpf-rust-spoof1"
 version = "1.9.6"

programs/bpf/Cargo.toml

@@ -81,6 +81,8 @@ members = [
     "rust/sanity",
     "rust/secp256k1_recover",
     "rust/sha",
+    "rust/sibling_inner_instruction",
+    "rust/sibling_instruction",
     "rust/spoof1",
     "rust/spoof1_system",
     "rust/sysvar",

programs/bpf/build.rs

@@ -91,6 +91,8 @@ fn main() {
             "sanity",
             "secp256k1_recover",
             "sha",
+            "sibling_inner_instruction",
+            "sibling_instruction",
             "spoof1",
             "spoof1_system",
             "upgradeable",

programs/bpf/rust/sibling_inner_instruction/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "solana-bpf-rust-sibling_inner-instructions"
+version = "1.9.6"
+description = "Solana BPF test program written in Rust"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-bpf-rust-log-data"
+edition = "2021"
+
+[dependencies]
+solana-program = { path = "../../../../sdk/program", version = "=1.9.6" }
+
+[features]
+default = ["program"]
+program = []
+
+[lib]
+crate-type = ["lib", "cdylib"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

programs/bpf/rust/sibling_inner_instruction/src/lib.rs

@@ -0,0 +1,69 @@
+//! Example Rust-based BPF program that queries sibling instructions
+
+#![cfg(feature = "program")]
+
+use solana_program::{
+    account_info::AccountInfo,
+    entrypoint,
+    entrypoint::ProgramResult,
+    instruction::{
+        get_processed_sibling_instruction, get_stack_height, AccountMeta, Instruction,
+        TRANSACTION_LEVEL_STACK_HEIGHT,
+    },
+    msg,
+    pubkey::Pubkey,
+};
+
+entrypoint!(process_instruction);
+fn process_instruction(
+    _program_id: &Pubkey,
+    accounts: &[AccountInfo],
+    _instruction_data: &[u8],
+) -> ProgramResult {
+    msg!("sibling inner");
+
+    // account 0 is mint
+    // account 1 is noop
+    // account 2 is invoke_and_return
+
+    // Check sibling instructions
+
+    let sibling_instruction2 = Instruction::new_with_bytes(
+        *accounts[2].key,
+        &[3],
+        vec![AccountMeta::new_readonly(*accounts[1].key, false)],
+    );
+    let sibling_instruction1 = Instruction::new_with_bytes(
+        *accounts[1].key,
+        &[2],
+        vec![
+            AccountMeta::new_readonly(*accounts[0].key, true),
+            AccountMeta::new_readonly(*accounts[1].key, false),
+        ],
+    );
+    let sibling_instruction0 = Instruction::new_with_bytes(
+        *accounts[1].key,
+        &[1],
+        vec![
+            AccountMeta::new_readonly(*accounts[1].key, false),
+            AccountMeta::new_readonly(*accounts[0].key, true),
+        ],
+    );
+
+    assert_eq!(TRANSACTION_LEVEL_STACK_HEIGHT + 1, get_stack_height());
+    assert_eq!(
+        get_processed_sibling_instruction(0),
+        Some(sibling_instruction0)
+    );
+    assert_eq!(
+        get_processed_sibling_instruction(1),
+        Some(sibling_instruction1)
+    );
+    assert_eq!(
+        get_processed_sibling_instruction(2),
+        Some(sibling_instruction2)
+    );
+    assert_eq!(get_processed_sibling_instruction(3), None);
+
+    Ok(())
+}

programs/bpf/rust/sibling_instruction/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "solana-bpf-rust-sibling-instructions"
+version = "1.9.6"
+description = "Solana BPF test program written in Rust"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-bpf-rust-log-data"
+edition = "2021"
+
+[dependencies]
+solana-program = { path = "../../../../sdk/program", version = "=1.9.6" }
+
+[features]
+default = ["program"]
+program = []
+
+[lib]
+crate-type = ["lib", "cdylib"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

programs/bpf/rust/sibling_instruction/src/lib.rs

@@ -0,0 +1,100 @@
+//! Example Rust-based BPF program that queries sibling instructions
+
+#![cfg(feature = "program")]
+
+use solana_program::{
+    account_info::AccountInfo,
+    entrypoint,
+    entrypoint::ProgramResult,
+    instruction::{
+        get_processed_sibling_instruction, get_stack_height, AccountMeta, Instruction,
+        TRANSACTION_LEVEL_STACK_HEIGHT,
+    },
+    msg,
+    program::invoke,
+    pubkey::Pubkey,
+};
+
+entrypoint!(process_instruction);
+fn process_instruction(
+    _program_id: &Pubkey,
+    accounts: &[AccountInfo],
+    _instruction_data: &[u8],
+) -> ProgramResult {
+    msg!("sibling");
+
+    // account 0 is mint
+    // account 1 is noop
+    // account 2 is invoke_and_return
+    // account 3 is sibling_inner
+
+    // Invoke child instructions
+
+    let instruction3 = Instruction::new_with_bytes(
+        *accounts[2].key,
+        &[3],
+        vec![AccountMeta::new_readonly(*accounts[1].key, false)],
+    );
+    let instruction2 = Instruction::new_with_bytes(
+        *accounts[1].key,
+        &[2],
+        vec![
+            AccountMeta::new_readonly(*accounts[0].key, true),
+            AccountMeta::new_readonly(*accounts[1].key, false),
+        ],
+    );
+    let instruction1 = Instruction::new_with_bytes(
+        *accounts[1].key,
+        &[1],
+        vec![
+            AccountMeta::new_readonly(*accounts[1].key, false),
+            AccountMeta::new_readonly(*accounts[0].key, true),
+        ],
+    );
+    let instruction0 = Instruction::new_with_bytes(
+        *accounts[3].key,
+        &[0],
+        vec![
+            AccountMeta::new_readonly(*accounts[0].key, false),
+            AccountMeta::new_readonly(*accounts[1].key, false),
+            AccountMeta::new_readonly(*accounts[2].key, false),
+            AccountMeta::new_readonly(*accounts[3].key, false),
+        ],
+    );
+    invoke(&instruction3, accounts)?;
+    invoke(&instruction2, accounts)?;
+    invoke(&instruction1, accounts)?;
+    invoke(&instruction0, accounts)?;
+
+    // Check sibling instructions
+
+    let sibling_instruction1 = Instruction::new_with_bytes(
+        *accounts[1].key,
+        &[43],
+        vec![
+            AccountMeta::new_readonly(*accounts[1].key, false),
+            AccountMeta::new(*accounts[0].key, true),
+        ],
+    );
+    let sibling_instruction0 = Instruction::new_with_bytes(
+        *accounts[1].key,
+        &[42],
+        vec![
+            AccountMeta::new(*accounts[0].key, true),
+            AccountMeta::new_readonly(*accounts[1].key, false),
+        ],
+    );
+
+    assert_eq!(TRANSACTION_LEVEL_STACK_HEIGHT, get_stack_height());
+    assert_eq!(
+        get_processed_sibling_instruction(0),
+        Some(sibling_instruction0)
+    );
+    assert_eq!(
+        get_processed_sibling_instruction(1),
+        Some(sibling_instruction1)
+    );
+    assert_eq!(get_processed_sibling_instruction(2), None);
+
+    Ok(())
+}

programs/bpf/tests/programs.rs

@@ -3327,3 +3327,79 @@ fn test_program_bpf_realloc_invoke() {
         TransactionError::InstructionError(0, InstructionError::InvalidRealloc)
     );
 }
+
+#[test]
+#[cfg(any(feature = "bpf_rust"))]
+fn test_program_bpf_processed_inner_instruction() {
+    solana_logger::setup();
+
+    let GenesisConfigInfo {
+        genesis_config,
+        mint_keypair,
+        ..
+    } = create_genesis_config(50);
+    let mut bank = Bank::new_for_tests(&genesis_config);
+    let (name, id, entrypoint) = solana_bpf_loader_program!();
+    bank.add_builtin(&name, &id, entrypoint);
+    let bank = Arc::new(bank);
+    let bank_client = BankClient::new_shared(&bank);
+
+    let sibling_program_id = load_bpf_program(
+        &bank_client,
+        &bpf_loader::id(),
+        &mint_keypair,
+        "solana_bpf_rust_sibling_instructions",
+    );
+    let sibling_inner_program_id = load_bpf_program(
+        &bank_client,
+        &bpf_loader::id(),
+        &mint_keypair,
+        "solana_bpf_rust_sibling_inner_instructions",
+    );
+    let noop_program_id = load_bpf_program(
+        &bank_client,
+        &bpf_loader::id(),
+        &mint_keypair,
+        "solana_bpf_rust_noop",
+    );
+    let invoke_and_return_program_id = load_bpf_program(
+        &bank_client,
+        &bpf_loader::id(),
+        &mint_keypair,
+        "solana_bpf_rust_invoke_and_return",
+    );
+
+    let instruction2 = Instruction::new_with_bytes(
+        noop_program_id,
+        &[43],
+        vec![
+            AccountMeta::new_readonly(noop_program_id, false),
+            AccountMeta::new(mint_keypair.pubkey(), true),
+        ],
+    );
+    let instruction1 = Instruction::new_with_bytes(
+        noop_program_id,
+        &[42],
+        vec![
+            AccountMeta::new(mint_keypair.pubkey(), true),
+            AccountMeta::new_readonly(noop_program_id, false),
+        ],
+    );
+    let instruction0 = Instruction::new_with_bytes(
+        sibling_program_id,
+        &[1, 2, 3, 0, 4, 5, 6],
+        vec![
+            AccountMeta::new(mint_keypair.pubkey(), true),
+            AccountMeta::new_readonly(noop_program_id, false),
+            AccountMeta::new_readonly(invoke_and_return_program_id, false),
+            AccountMeta::new_readonly(sibling_inner_program_id, false),
+        ],
+    );
+    let message = Message::new(
+        &[instruction2, instruction1, instruction0],
+        Some(&mint_keypair.pubkey()),
+    );
+    assert!(bank_client
+        .send_and_confirm_message(&[&mint_keypair], message)
+        .is_ok());
+}

programs/bpf_loader/src/lib.rs

@@ -298,7 +298,7 @@ fn process_instruction_common(
     if program.executable()? {
         debug_assert_eq!(
             first_instruction_account,
-            1 - (invoke_context.invoke_depth() > 1) as usize,
+            1 - (invoke_context.get_stack_height() > 1) as usize,
         );
 
         if !check_loader_id(&program.owner()?) {
@@ -1036,7 +1036,7 @@ impl Executor for BpfExecutor {
     ) -> Result<(), InstructionError> {
         let log_collector = invoke_context.get_log_collector();
         let compute_meter = invoke_context.get_compute_meter();
-        let invoke_depth = invoke_context.invoke_depth();
+        let stack_height = invoke_context.get_stack_height();
 
         let mut serialize_time = Measure::start("serialize");
         let keyed_accounts = invoke_context.get_keyed_accounts()?;
@@ -1068,7 +1068,7 @@ impl Executor for BpfExecutor {
             create_vm_time.stop();
 
             execute_time = Measure::start("execute");
-            stable_log::program_invoke(&log_collector, &program_id, invoke_depth);
+            stable_log::program_invoke(&log_collector, &program_id, stack_height);
             let mut instruction_meter = ThisInstructionMeter::new(compute_meter.clone());
             let before = compute_meter.borrow().get_remaining();
             let result = if use_jit {

programs/bpf_loader/src/syscalls.rs

@@ -21,13 +21,17 @@ use {
         blake3, bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable,
         entrypoint::{BPF_ALIGN_OF_U128, MAX_PERMITTED_DATA_INCREASE, SUCCESS},
         feature_set::{
-            blake3_syscall_enabled, disable_fees_sysvar, do_support_realloc,
+            add_get_processed_sibling_instruction_syscall, blake3_syscall_enabled,
-            libsecp256k1_0_5_upgrade_enabled, prevent_calling_precompiles_as_programs,
+            disable_fees_sysvar, do_support_realloc, libsecp256k1_0_5_upgrade_enabled,
-            return_data_syscall_enabled, secp256k1_recover_syscall_enabled,
+            prevent_calling_precompiles_as_programs, return_data_syscall_enabled,
-            sol_log_data_syscall_enabled, update_syscall_base_costs,
+            secp256k1_recover_syscall_enabled, sol_log_data_syscall_enabled,
+            update_syscall_base_costs,
         },
         hash::{Hasher, HASH_BYTES},
-        instruction::{AccountMeta, Instruction, InstructionError},
+        instruction::{
+            AccountMeta, Instruction, InstructionError, ProcessedSiblingInstruction,
+            TRANSACTION_LEVEL_STACK_HEIGHT,
+        },
         keccak,
         message::{Message, SanitizedMessage},
         native_loader,
@@ -204,6 +208,24 @@ pub fn register_syscalls(
         syscall_registry.register_syscall_by_name(b"sol_log_data", SyscallLogData::call)?;
     }
 
+    if invoke_context
+        .feature_set
+        .is_active(&add_get_processed_sibling_instruction_syscall::id())
+    {
+        syscall_registry.register_syscall_by_name(
+            b"sol_get_processed_sibling_instruction",
+            SyscallGetProcessedSiblingInstruction::call,
+        )?;
+    }
+
+    if invoke_context
+        .feature_set
+        .is_active(&add_get_processed_sibling_instruction_syscall::id())
+    {
+        syscall_registry
+            .register_syscall_by_name(b"sol_get_stack_height", SyscallGetStackHeight::call)?;
+    }
+
     Ok(syscall_registry)
 }
 
@@ -241,6 +263,9 @@ pub fn bind_syscall_context_objects<'a, 'b>(
     let is_sol_log_data_syscall_active = invoke_context
         .feature_set
         .is_active(&sol_log_data_syscall_enabled::id());
+    let add_get_processed_sibling_instruction_syscall = invoke_context
+        .feature_set
+        .is_active(&add_get_processed_sibling_instruction_syscall::id());
 
     let loader_id = invoke_context
         .get_loader()
@@ -400,6 +425,24 @@ pub fn bind_syscall_context_objects<'a, 'b>(
         }),
     );
 
+    // processed inner instructions
+    bind_feature_gated_syscall_context_object!(
+        vm,
+        add_get_processed_sibling_instruction_syscall,
+        Box::new(SyscallGetProcessedSiblingInstruction {
+            invoke_context: invoke_context.clone(),
+        }),
+    );
+
+    // Get stack height
+    bind_feature_gated_syscall_context_object!(
+        vm,
+        add_get_processed_sibling_instruction_syscall,
+        Box::new(SyscallGetStackHeight {
+            invoke_context: invoke_context.clone(),
+        }),
+    );
+
     // Cross-program invocation syscalls
     vm.bind_syscall_context_object(
         Box::new(SyscallInvokeSignedC {
@@ -2387,6 +2430,7 @@ fn call<'a, 'b: 'a>(
         signers_seeds_len,
         memory_mapping,
     )?;
+    let stack_height = invoke_context.get_stack_height();
     let (message, caller_write_privileges, program_indices) = invoke_context
         .create_message(&instruction, &signers)
         .map_err(SyscallError::InstructionError)?;
@@ -2401,9 +2445,7 @@ fn call<'a, 'b: 'a>(
     )?;
 
     // Record the instruction
-    if let Some(instruction_recorder) = &invoke_context.instruction_recorder {
+    invoke_context.record_instruction(stack_height.saturating_add(1), instruction);
-        instruction_recorder.record_instruction(instruction);
-    }
 
     // Process instruction
     let message = SanitizedMessage::Legacy(message);
@@ -2684,6 +2726,141 @@ impl<'a, 'b> SyscallObject<BpfError> for SyscallLogData<'a, 'b> {
     }
 }
 
+pub struct SyscallGetProcessedSiblingInstruction<'a, 'b> {
+    invoke_context: Rc<RefCell<&'a mut InvokeContext<'b>>>,
+}
+impl<'a, 'b> SyscallObject<BpfError> for SyscallGetProcessedSiblingInstruction<'a, 'b> {
+    fn call(
+        &mut self,
+        index: u64,
+        meta_addr: u64,
+        program_id_addr: u64,
+        data_addr: u64,
+        accounts_addr: u64,
+        memory_mapping: &MemoryMapping,
+        result: &mut Result<u64, EbpfError<BpfError>>,
+    ) {
+        let invoke_context = question_mark!(
+            self.invoke_context
+                .try_borrow()
+                .map_err(|_| SyscallError::InvokeContextBorrowFailed),
+            result
+        );
+        let loader_id = question_mark!(
+            invoke_context
+                .get_loader()
+                .map_err(SyscallError::InstructionError),
+            result
+        );
+
+        let budget = invoke_context.get_compute_budget();
+        question_mark!(
+            invoke_context
+                .get_compute_meter()
+                .consume(budget.syscall_base_cost),
+            result
+        );
+
+        let stack_height = invoke_context.get_stack_height();
+        let instruction_trace = invoke_context.get_instruction_trace();
+        let instruction = if stack_height == TRANSACTION_LEVEL_STACK_HEIGHT {
+            // pick one of the top-level instructions
+            instruction_trace
+                .len()
+                .checked_sub(2)
+                .and_then(|result| result.checked_sub(index as usize))
+                .and_then(|index| instruction_trace.get(index))
+                .and_then(|instruction_recorder| instruction_recorder.get(0))
+        } else {
+            // Walk the last list of inner instructions
+            instruction_trace
+                .last()
+                .and_then(|inners| inners.find(stack_height, index as usize))
+        };
+
+        if let Some(instruction) = instruction {
+            let ProcessedSiblingInstruction {
+                data_len,
+                accounts_len,
+            } = question_mark!(
+                translate_type_mut::<ProcessedSiblingInstruction>(
+                    memory_mapping,
+                    meta_addr,
+                    &loader_id
+                ),
+                result
+            );
+
+            if *data_len == instruction.data.len() && *accounts_len == instruction.accounts.len() {
+                let program_id = question_mark!(
+                    translate_type_mut::<Pubkey>(memory_mapping, program_id_addr, &loader_id),
+                    result
+                );
+                let data = question_mark!(
+                    translate_slice_mut::<u8>(
+                        memory_mapping,
+                        data_addr,
+                        *data_len as u64,
+                        &loader_id,
+                    ),
+                    result
+                );
+                let accounts = question_mark!(
+                    translate_slice_mut::<AccountMeta>(
+                        memory_mapping,
+                        accounts_addr,
+                        *accounts_len as u64,
+                        &loader_id,
+                    ),
+                    result
+                );
+
+                *program_id = instruction.program_id;
+                data.clone_from_slice(instruction.data.as_slice());
+                accounts.clone_from_slice(instruction.accounts.as_slice());
+            }
+            *data_len = instruction.data.len();
+            *accounts_len = instruction.accounts.len();
+            *result = Ok(true as u64);
+            return;
+        }
+        *result = Ok(false as u64);
+    }
+}
+
+pub struct SyscallGetStackHeight<'a, 'b> {
+    invoke_context: Rc<RefCell<&'a mut InvokeContext<'b>>>,
+}
+impl<'a, 'b> SyscallObject<BpfError> for SyscallGetStackHeight<'a, 'b> {
+    fn call(
+        &mut self,
+        _arg1: u64,
+        _arg2: u64,
+        _arg3: u64,
+        _arg4: u64,
+        _arg5: u64,
+        _memory_mapping: &MemoryMapping,
+        result: &mut Result<u64, EbpfError<BpfError>>,
+    ) {
+        let invoke_context = question_mark!(
+            self.invoke_context
+                .try_borrow()
+                .map_err(|_| SyscallError::InvokeContextBorrowFailed),
+            result
+        );
+
+        let budget = invoke_context.get_compute_budget();
+        question_mark!(
+            invoke_context
+                .get_compute_meter()
+                .consume(budget.syscall_base_cost),
+            result
+        );
+
+        *result = Ok(invoke_context.get_stack_height() as u64);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     #[allow(deprecated)]

runtime/src/bank.rs

@@ -53,7 +53,7 @@ use {
         cost_tracker::CostTracker,
         epoch_stakes::{EpochStakes, NodeVoteAccounts},
         inline_spl_associated_token_account, inline_spl_token,
-        message_processor::MessageProcessor,
+        message_processor::{InstructionTrace, MessageProcessor},
         rent_collector::{CollectedInfo, RentCollector},
         stake_weighted_timestamp::{
             calculate_stake_weighted_timestamp, MaxAllowableDrift, MAX_ALLOWABLE_DRIFT_PERCENTAGE,
@@ -79,7 +79,6 @@ use {
     solana_metrics::{inc_new_counter_debug, inc_new_counter_info},
     solana_program_runtime::{
         compute_budget::ComputeBudget,
-        instruction_recorder::InstructionRecorder,
         invoke_context::{
             BuiltinProgram, Executor, Executors, ProcessInstructionWithContext,
             TransactionAccountRefCells, TransactionExecutor,
@@ -696,6 +695,17 @@ pub type InnerInstructions = Vec<CompiledInstruction>;
 /// A list of instructions that were invoked during each instruction of a transaction
 pub type InnerInstructionsList = Vec<InnerInstructions>;
 
+/// Convert from an InstructionTrace to InnerInstructionsList
+pub fn inner_instructions_list_from_instruction_trace(
+    instruction_trace: &InstructionTrace,
+    message: &SanitizedMessage,
+) -> Option<InnerInstructionsList> {
+    instruction_trace
+        .iter()
+        .map(|r| r.compile_instructions(message))
+        .collect()
+}
+
 /// A list of log messages emitted during a transaction
 pub type TransactionLogMessages = Vec<String>;
 
@@ -3880,15 +3890,7 @@ impl Bank {
         let pre_account_state_info =
             self.get_transaction_account_state_info(&account_refcells, tx.message());
 
-        let instruction_recorders = if enable_cpi_recording {
+        let mut instruction_trace = Vec::with_capacity(tx.message().instructions().len());
-            let ix_count = tx.message().instructions().len();
-            let mut recorders = Vec::with_capacity(ix_count);
-            recorders.resize_with(ix_count, InstructionRecorder::default);
-            Some(recorders)
-        } else {
-            None
-        };
-
         let log_collector = if enable_log_recording {
             Some(LogCollector::new_ref())
         } else {
@@ -3906,7 +3908,7 @@ impl Bank {
             self.rent_collector.rent,
             log_collector.clone(),
             executors.clone(),
-            instruction_recorders.as_deref(),
+            &mut instruction_trace,
             self.feature_set.clone(),
             compute_budget,
             timings,
@@ -3962,13 +3964,11 @@ impl Bank {
                     .ok()
             });
 
-        let inner_instructions: Option<InnerInstructionsList> =
+        let inner_instructions = if enable_cpi_recording {
-            instruction_recorders.and_then(|instruction_recorders| {
+            inner_instructions_list_from_instruction_trace(&instruction_trace, tx.message())
-                instruction_recorders
+        } else {
-                    .into_iter()
+            None
-                    .map(|r| r.compile_instructions(tx.message()))
+        };
-                    .collect()
-            });
 
         if let Err(e) =
             Self::refcells_to_accounts(&mut loaded_transaction.accounts, account_refcells)
@@ -6677,7 +6677,9 @@ pub(crate) mod tests {
             status_cache::MAX_CACHE_ENTRIES,
         },
         crossbeam_channel::{bounded, unbounded},
-        solana_program_runtime::invoke_context::InvokeContext,
+        solana_program_runtime::{
+            instruction_recorder::InstructionRecorder, invoke_context::InvokeContext,
+        },
         solana_sdk::{
             account::Account,
             bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable,
@@ -16172,4 +16174,56 @@ pub(crate) mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_inner_instructions_list_from_instruction_trace() {
+        let instruction1 = Instruction::new_with_bytes(Pubkey::default(), &[1], Vec::new());
+        let instruction2 = Instruction::new_with_bytes(Pubkey::default(), &[2], Vec::new());
+        let instruction3 = Instruction::new_with_bytes(Pubkey::default(), &[3], Vec::new());
+        let instruction4 = Instruction::new_with_bytes(Pubkey::default(), &[4], Vec::new());
+        let instruction5 = Instruction::new_with_bytes(Pubkey::default(), &[5], Vec::new());
+        let instruction6 = Instruction::new_with_bytes(Pubkey::default(), &[6], Vec::new());
+
+        let instruction_trace = vec![
+            InstructionRecorder::default(),
+            InstructionRecorder::default(),
+            InstructionRecorder::default(),
+        ];
+        instruction_trace[0].record_instruction(1, instruction1.clone());
+        instruction_trace[0].record_instruction(2, instruction2.clone());
+        instruction_trace[2].record_instruction(1, instruction3.clone());
+        instruction_trace[2].record_instruction(2, instruction4.clone());
+        instruction_trace[2].record_instruction(3, instruction5.clone());
+        instruction_trace[2].record_instruction(2, instruction6.clone());
+
+        let message = Message::new(
+            &[
+                instruction1,
+                instruction2,
+                instruction3,
+                instruction4,
+                instruction5,
+                instruction6,
+            ],
+            None,
+        );
+
+        let inner_instructions = inner_instructions_list_from_instruction_trace(
+            &instruction_trace,
+            &SanitizedMessage::Legacy(message),
+        );
+
+        assert_eq!(
+            inner_instructions,
+            Some(vec![
+                vec![CompiledInstruction::new_from_raw_parts(0, vec![2], vec![])],
+                vec![],
+                vec![
+                    CompiledInstruction::new_from_raw_parts(0, vec![4], vec![]),
+                    CompiledInstruction::new_from_raw_parts(0, vec![5], vec![]),
+                    CompiledInstruction::new_from_raw_parts(0, vec![6], vec![])
+                ]
+            ])
+        );
+    }
 }

runtime/src/builtins.rs

@@ -20,7 +20,7 @@ fn process_instruction_with_program_logging(
 ) -> Result<(), InstructionError> {
     let logger = invoke_context.get_log_collector();
     let program_id = invoke_context.get_caller()?;
-    stable_log::program_invoke(&logger, program_id, invoke_context.invoke_depth());
+    stable_log::program_invoke(&logger, program_id, invoke_context.get_stack_height());
 
     let result = process_instruction(first_instruction_account, instruction_data, invoke_context);
 

runtime/src/message_processor.rs

@@ -38,6 +38,9 @@ impl ::solana_frozen_abi::abi_example::AbiExample for MessageProcessor {
     }
 }
 
+/// Trace of all instructions attempted
+pub type InstructionTrace = Vec<InstructionRecorder>;
+
 /// Resultant information gathered from calling process_message()
 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub struct ProcessedMessageInfo {
@@ -60,7 +63,7 @@ impl MessageProcessor {
         rent: Rent,
         log_collector: Option<Rc<RefCell<LogCollector>>>,
         executors: Rc<RefCell<Executors>>,
-        instruction_recorders: Option<&[InstructionRecorder]>,
+        instruction_trace: &mut InstructionTrace,
         feature_set: Arc<FeatureSet>,
         compute_budget: ComputeBudget,
         timings: &mut ExecuteTimings,
@@ -89,6 +92,12 @@ impl MessageProcessor {
             .zip(program_indices.iter())
             .enumerate()
         {
+            invoke_context.record_top_level_instruction(
+                instruction.decompile(message).map_err(|err| {
+                    TransactionError::InstructionError(instruction_index as u8, err)
+                })?,
+            );
+
             if invoke_context
                 .feature_set
                 .is_active(&prevent_calling_precompiles_as_programs::id())
@@ -111,10 +120,6 @@ impl MessageProcessor {
                 }
             }
 
-            if let Some(instruction_recorders) = instruction_recorders {
-                invoke_context.instruction_recorder =
-                    Some(&instruction_recorders[instruction_index]);
-            }
             let mut time = Measure::start("execute_instruction");
             let ProcessInstructionResult {
                 compute_units_consumed,
@@ -139,9 +144,13 @@ impl MessageProcessor {
                 timings.execute_accessories.process_instructions.total_us,
                 time.as_us()
             );
-            result
+
-                .map_err(|err| TransactionError::InstructionError(instruction_index as u8, err))?;
+            result.map_err(|err| {
+                instruction_trace.append(invoke_context.get_instruction_trace_mut());
+                TransactionError::InstructionError(instruction_index as u8, err)
+            })?;
         }
+        instruction_trace.append(invoke_context.get_instruction_trace_mut());
         Ok(ProcessedMessageInfo {
             accounts_data_len: invoke_context.get_accounts_data_meter().current(),
         })
@@ -263,7 +272,7 @@ mod tests {
             rent_collector.rent,
             None,
             executors.clone(),
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),
@@ -293,7 +302,7 @@ mod tests {
             rent_collector.rent,
             None,
             executors.clone(),
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),
@@ -327,7 +336,7 @@ mod tests {
             rent_collector.rent,
             None,
             executors,
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),
@@ -473,7 +482,7 @@ mod tests {
             rent_collector.rent,
             None,
             executors.clone(),
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),
@@ -507,7 +516,7 @@ mod tests {
             rent_collector.rent,
             None,
             executors.clone(),
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),
@@ -538,7 +547,7 @@ mod tests {
             rent_collector.rent,
             None,
             executors,
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),
@@ -596,7 +605,7 @@ mod tests {
             RentCollector::default().rent,
             None,
             Rc::new(RefCell::new(Executors::default())),
-            None,
+            &mut Vec::new(),
             Arc::new(FeatureSet::all_enabled()),
             ComputeBudget::new(),
             &mut ExecuteTimings::default(),

sdk/program/src/instruction.rs

@@ -528,7 +528,8 @@ pub fn checked_add(a: u64, b: u64) -> Result<u64, InstructionError> {
 /// default [`AccountMeta::new`] constructor creates writable accounts, this is
 /// a minor hazard: use [`AccountMeta::new_readonly`] to specify that an account
 /// is not writable.
-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+#[repr(C)]
+#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
 pub struct AccountMeta {
     /// An account's public key.
     pub pubkey: Pubkey,
@@ -639,8 +640,16 @@ impl CompiledInstruction {
         let data = serialize(data).unwrap();
         Self {
             program_id_index: program_ids_index,
-            data,
             accounts,
+            data,
+        }
+    }
+
+    pub fn new_from_raw_parts(program_id_index: u8, data: Vec<u8>, accounts: Vec<u8>) -> Self {
+        Self {
+            program_id_index,
+            accounts,
+            data,
         }
     }
 
@@ -667,6 +676,32 @@ impl CompiledInstruction {
         }
         Ok(())
     }
+
+    #[cfg(not(target_arch = "bpf"))]
+    pub fn decompile(
+        &self,
+        message: &crate::message::SanitizedMessage,
+    ) -> Result<Instruction, InstructionError> {
+        Ok(Instruction::new_with_bytes(
+            *message
+                .get_account_key(self.program_id_index as usize)
+                .ok_or(InstructionError::MissingAccount)?,
+            &self.data,
+            self.accounts
+                .iter()
+                .map(|account_index| {
+                    let account_index = *account_index as usize;
+                    Ok(AccountMeta {
+                        is_signer: message.is_signer(account_index),
+                        is_writable: message.is_writable(account_index),
+                        pubkey: *message
+                            .get_account_key(account_index)
+                            .ok_or(InstructionError::MissingAccount)?,
+                    })
+                })
+                .collect::<Result<Vec<AccountMeta>, InstructionError>>()?,
+        ))
+    }
 }
 
 #[cfg(test)]
@@ -696,3 +731,137 @@ mod test {
         assert_eq!((0, 2), do_work(&[2, 2]));
     }
 }
+
+/// Use to query and convey information about the sibling instruction components
+/// when calling the `sol_get_processed_sibling_instruction` syscall.
+#[repr(C)]
+#[derive(Default, Debug, Clone, Copy)]
+pub struct ProcessedSiblingInstruction {
+    /// Length of the instruction data
+    pub data_len: usize,
+    /// Number of AccountMeta structures
+    pub accounts_len: usize,
+}
+
+/// Returns a sibling instruction from the processed sibling instruction list.
+///
+/// The processed sibling instruction list is a reverse-ordered list of
+/// successfully processed sibling instructions. For example, given the call flow:
+///
+/// A
+/// B -> C -> D
+/// B -> E
+/// B -> F
+///
+/// Then B's processed sibling instruction list is: `[A]`
+/// Then F's processed sibling instruction list is: `[E, C]`
+pub fn get_processed_sibling_instruction(index: usize) -> Option<Instruction> {
+    #[cfg(target_arch = "bpf")]
+    {
+        extern "C" {
+            fn sol_get_processed_sibling_instruction(
+                index: u64,
+                meta: *mut ProcessedSiblingInstruction,
+                program_id: *mut Pubkey,
+                data: *mut u8,
+                accounts: *mut AccountMeta,
+            ) -> u64;
+        }
+
+        let mut meta = ProcessedSiblingInstruction::default();
+        let mut program_id = Pubkey::default();
+
+        if 1 == unsafe {
+            sol_get_processed_sibling_instruction(
+                index as u64,
+                &mut meta,
+                &mut program_id,
+                &mut u8::default(),
+                &mut AccountMeta::default(),
+            )
+        } {
+            let mut data = Vec::new();
+            let mut accounts = Vec::new();
+            data.resize_with(meta.data_len, u8::default);
+            accounts.resize_with(meta.accounts_len, AccountMeta::default);
+
+            let _ = unsafe {
+                sol_get_processed_sibling_instruction(
+                    index as u64,
+                    &mut meta,
+                    &mut program_id,
+                    data.as_mut_ptr(),
+                    accounts.as_mut_ptr(),
+                )
+            };
+
+            Some(Instruction::new_with_bytes(program_id, &data, accounts))
+        } else {
+            None
+        }
+    }
+
+    #[cfg(not(target_arch = "bpf"))]
+    crate::program_stubs::sol_get_processed_sibling_instruction(index)
+}
+
+// Stack height when processing transaction-level instructions
+pub const TRANSACTION_LEVEL_STACK_HEIGHT: usize = 1;
+
+/// Get the current stack height, transaction-level instructions are height
+/// TRANSACTION_LEVEL_STACK_HEIGHT, fist invoked inner instruction is height
+/// TRANSACTION_LEVEL_STACK_HEIGHT + 1, etc...
+pub fn get_stack_height() -> usize {
+    #[cfg(target_arch = "bpf")]
+    {
+        extern "C" {
+            fn sol_get_stack_height() -> u64;
+        }
+
+        unsafe { sol_get_stack_height() as usize }
+    }
+
+    #[cfg(not(target_arch = "bpf"))]
+    {
+        crate::program_stubs::sol_get_stack_height() as usize
+    }
+}
+
+#[test]
+fn test_account_meta_layout() {
+    #[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
+    struct AccountMetaRust {
+        pub pubkey: Pubkey,
+        pub is_signer: bool,
+        pub is_writable: bool,
+    }
+
+    let account_meta_rust = AccountMetaRust::default();
+    let base_rust_addr = &account_meta_rust as *const _ as u64;
+    let pubkey_rust_addr = &account_meta_rust.pubkey as *const _ as u64;
+    let is_signer_rust_addr = &account_meta_rust.is_signer as *const _ as u64;
+    let is_writable_rust_addr = &account_meta_rust.is_writable as *const _ as u64;
+
+    let account_meta_c = AccountMeta::default();
+    let base_c_addr = &account_meta_c as *const _ as u64;
+    let pubkey_c_addr = &account_meta_c.pubkey as *const _ as u64;
+    let is_signer_c_addr = &account_meta_c.is_signer as *const _ as u64;
+    let is_writable_c_addr = &account_meta_c.is_writable as *const _ as u64;
+
+    assert_eq!(
+        std::mem::size_of::<AccountMetaRust>(),
+        std::mem::size_of::<AccountMeta>()
+    );
+    assert_eq!(
+        pubkey_rust_addr - base_rust_addr,
+        pubkey_c_addr - base_c_addr
+    );
+    assert_eq!(
+        is_signer_rust_addr - base_rust_addr,
+        is_signer_c_addr - base_c_addr
+    );
+    assert_eq!(
+        is_writable_rust_addr - base_rust_addr,
+        is_writable_c_addr - base_c_addr
+    );
+}

sdk/program/src/program_stubs.rs

@@ -91,6 +91,12 @@ pub trait SyscallStubs: Sync + Send {
     fn sol_log_data(&self, fields: &[&[u8]]) {
         println!("data: {}", fields.iter().map(base64::encode).join(" "));
     }
+    fn sol_get_processed_sibling_instruction(&self, _index: usize) -> Option<Instruction> {
+        None
+    }
+    fn sol_get_stack_height(&self) -> u64 {
+        0
+    }
 }
 
 struct DefaultSyscallStubs {}
@@ -176,3 +182,14 @@ pub(crate) fn sol_set_return_data(data: &[u8]) {
 pub(crate) fn sol_log_data(data: &[&[u8]]) {
     SYSCALL_STUBS.read().unwrap().sol_log_data(data)
 }
+
+pub(crate) fn sol_get_processed_sibling_instruction(index: usize) -> Option<Instruction> {
+    SYSCALL_STUBS
+        .read()
+        .unwrap()
+        .sol_get_processed_sibling_instruction(index)
+}
+
+pub(crate) fn sol_get_stack_height() -> u64 {
+    SYSCALL_STUBS.read().unwrap().sol_get_stack_height()
+}

sdk/src/feature_set.rs

@@ -295,6 +295,10 @@ pub mod disable_bpf_unresolved_symbols_at_runtime {
     solana_sdk::declare_id!("4yuaYAj2jGMGTh1sSmi4G2eFscsDq8qjugJXZoBN6YEa");
 }
 
+pub mod add_get_processed_sibling_instruction_syscall {
+    solana_sdk::declare_id!("CFK1hRCNy8JJuAAY8Pb2GjLFNdCThS2qwZNe3izzBMgn");
+}
+
 lazy_static! {
     /// Map of feature identifiers to user-visible description
     pub static ref FEATURE_NAMES: HashMap<Pubkey, &'static str> = [
@@ -363,6 +367,7 @@ lazy_static! {
         (bank_tranaction_count_fix::id(), "Fixes Bank::transaction_count to include all committed transactions, not just successful ones"),
         (disable_bpf_deprecated_load_instructions::id(), "Disable ldabs* and ldind* BPF instructions"),
         (disable_bpf_unresolved_symbols_at_runtime::id(), "Disable reporting of unresolved BPF symbols at runtime"),
+        (add_get_processed_sibling_instruction_syscall::id(), "add add_get_processed_sibling_instruction_syscall"),
         /*************** ADD NEW FEATURES HERE ***************/
     ]
     .iter()