Replaces MockInvokeContext by ThisInvokeContext in tests (#20881)

* Replaces MockInvokeContext by ThisInvokeContext in BpfLoader, SystemInstructionProcessor, CLIs, ConfigProcessor, StakeProcessor and VoteProcessor.

* Finally, removes MockInvokeContext, MockComputeMeter and MockLogger.

* Adjusts assert_instruction_count test.

* Moves ThisInvokeContext to the program-runtime crate.
This commit is contained in:
Alexander Meißner
2021-11-04 21:47:32 +01:00
committed by GitHub
parent 0597594943
commit 7200c5106e
33 changed files with 2640 additions and 2783 deletions

View File

@@ -2566,6 +2566,7 @@ dependencies = [
"solana-cli-output",
"solana-logger 1.9.0",
"solana-measure",
"solana-program-runtime",
"solana-runtime",
"solana-sdk",
"solana-transaction-status",
@@ -3230,6 +3231,7 @@ dependencies = [
name = "solana-program-runtime"
version = "1.9.0"
dependencies = [
"bincode",
"libc",
"libloading",
"log",
@@ -3438,6 +3440,7 @@ dependencies = [
"solana-frozen-abi 1.9.0",
"solana-frozen-abi-macro 1.9.0",
"solana-metrics",
"solana-program-runtime",
"solana-sdk",
"solana-vote-program",
"thiserror",

View File

@@ -35,6 +35,7 @@ solana-logger = { path = "../../logger", version = "=1.9.0" }
solana-measure = { path = "../../measure", version = "=1.9.0" }
solana_rbpf = "=0.2.14"
solana-runtime = { path = "../../runtime", version = "=1.9.0" }
solana-program-runtime = { path = "../../program-runtime", version = "=1.9.0" }
solana-sdk = { path = "../../sdk", version = "=1.9.0" }
solana-transaction-status = { path = "../../transaction-status", version = "=1.9.0" }
solana-account-decoder = { path = "../../account-decoder", version = "=1.9.0" }

View File

@@ -18,18 +18,18 @@ use solana_runtime::{
genesis_utils::{create_genesis_config, GenesisConfigInfo},
loader_utils::load_program,
};
use solana_program_runtime::invoke_context::with_mock_invoke_context;
use solana_sdk::{
account::AccountSharedData,
bpf_loader,
client::SyncClient,
entrypoint::SUCCESS,
instruction::{AccountMeta, Instruction},
message::Message,
process_instruction::{InvokeContext, MockComputeMeter, MockInvokeContext},
process_instruction::InvokeContext,
pubkey::Pubkey,
signature::{Keypair, Signer},
};
use std::{cell::RefCell, env, fs::File, io::Read, mem, path::PathBuf, rc::Rc, sync::Arc};
use std::{env, fs::File, io::Read, mem, path::PathBuf, sync::Arc};
use test::Bencher;
/// BPF program file extension
@@ -94,93 +94,73 @@ fn bench_program_alu(bencher: &mut Bencher) {
.write_u64::<LittleEndian>(ARMSTRONG_LIMIT)
.unwrap();
inner_iter.write_u64::<LittleEndian>(0).unwrap();
let loader_id = bpf_loader::id();
let program_id = solana_sdk::pubkey::new_rand();
let accounts = [
(
program_id,
RefCell::new(AccountSharedData::new(0, 0, &loader_id)),
),
(
solana_sdk::pubkey::new_rand(),
RefCell::new(AccountSharedData::new(
1,
10000001,
&solana_sdk::pubkey::new_rand(),
)),
),
];
let keyed_accounts: Vec<_> = accounts
.iter()
.map(|(key, account)| solana_sdk::keyed_account::KeyedAccount::new(&key, false, &account))
.collect();
let mut invoke_context = MockInvokeContext::new(&program_id, keyed_accounts);
let elf = load_elf("bench_alu").unwrap();
let mut executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&elf,
None,
Config::default(),
register_syscalls(&mut invoke_context).unwrap(),
)
.unwrap();
executable.jit_compile().unwrap();
let compute_meter = invoke_context.get_compute_meter();
let mut instruction_meter = ThisInstructionMeter { compute_meter };
let mut vm = create_vm(
&loader_id,
executable.as_ref(),
&mut inner_iter,
&mut invoke_context,
&[],
)
.unwrap();
let loader_id = bpf_loader::id();
with_mock_invoke_context(loader_id, 10000001, |invoke_context| {
let mut executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&elf,
None,
Config::default(),
register_syscalls(invoke_context).unwrap(),
)
.unwrap();
executable.jit_compile().unwrap();
let compute_meter = invoke_context.get_compute_meter();
let mut instruction_meter = ThisInstructionMeter { compute_meter };
let mut vm = create_vm(
&loader_id,
executable.as_ref(),
&mut inner_iter,
invoke_context,
&[],
)
.unwrap();
println!("Interpreted:");
assert_eq!(
SUCCESS,
vm.execute_program_interpreted(&mut instruction_meter)
.unwrap()
);
assert_eq!(ARMSTRONG_LIMIT, LittleEndian::read_u64(&inner_iter));
assert_eq!(
ARMSTRONG_EXPECTED,
LittleEndian::read_u64(&inner_iter[mem::size_of::<u64>()..])
);
println!("Interpreted:");
assert_eq!(
SUCCESS,
vm.execute_program_interpreted(&mut instruction_meter)
.unwrap()
);
assert_eq!(ARMSTRONG_LIMIT, LittleEndian::read_u64(&inner_iter));
assert_eq!(
ARMSTRONG_EXPECTED,
LittleEndian::read_u64(&inner_iter[mem::size_of::<u64>()..])
);
bencher.iter(|| {
vm.execute_program_interpreted(&mut instruction_meter)
.unwrap();
bencher.iter(|| {
vm.execute_program_interpreted(&mut instruction_meter)
.unwrap();
});
let instructions = vm.get_total_instruction_count();
let summary = bencher.bench(|_bencher| {}).unwrap();
println!(" {:?} instructions", instructions);
println!(" {:?} ns/iter median", summary.median as u64);
assert!(0f64 != summary.median);
let mips = (instructions * (ns_per_s / summary.median as u64)) / one_million;
println!(" {:?} MIPS", mips);
println!("{{ \"type\": \"bench\", \"name\": \"bench_program_alu_interpreted_mips\", \"median\": {:?}, \"deviation\": 0 }}", mips);
println!("JIT to native:");
assert_eq!(
SUCCESS,
vm.execute_program_jit(&mut instruction_meter).unwrap()
);
assert_eq!(ARMSTRONG_LIMIT, LittleEndian::read_u64(&inner_iter));
assert_eq!(
ARMSTRONG_EXPECTED,
LittleEndian::read_u64(&inner_iter[mem::size_of::<u64>()..])
);
bencher.iter(|| vm.execute_program_jit(&mut instruction_meter).unwrap());
let summary = bencher.bench(|_bencher| {}).unwrap();
println!(" {:?} instructions", instructions);
println!(" {:?} ns/iter median", summary.median as u64);
assert!(0f64 != summary.median);
let mips = (instructions * (ns_per_s / summary.median as u64)) / one_million;
println!(" {:?} MIPS", mips);
println!("{{ \"type\": \"bench\", \"name\": \"bench_program_alu_jit_to_native_mips\", \"median\": {:?}, \"deviation\": 0 }}", mips);
});
let instructions = vm.get_total_instruction_count();
let summary = bencher.bench(|_bencher| {}).unwrap();
println!(" {:?} instructions", instructions);
println!(" {:?} ns/iter median", summary.median as u64);
assert!(0f64 != summary.median);
let mips = (instructions * (ns_per_s / summary.median as u64)) / one_million;
println!(" {:?} MIPS", mips);
println!("{{ \"type\": \"bench\", \"name\": \"bench_program_alu_interpreted_mips\", \"median\": {:?}, \"deviation\": 0 }}", mips);
println!("JIT to native:");
assert_eq!(
SUCCESS,
vm.execute_program_jit(&mut instruction_meter).unwrap()
);
assert_eq!(ARMSTRONG_LIMIT, LittleEndian::read_u64(&inner_iter));
assert_eq!(
ARMSTRONG_EXPECTED,
LittleEndian::read_u64(&inner_iter[mem::size_of::<u64>()..])
);
bencher.iter(|| vm.execute_program_jit(&mut instruction_meter).unwrap());
let summary = bencher.bench(|_bencher| {}).unwrap();
println!(" {:?} instructions", instructions);
println!(" {:?} ns/iter median", summary.median as u64);
assert!(0f64 != summary.median);
let mips = (instructions * (ns_per_s / summary.median as u64)) / one_million;
println!(" {:?} MIPS", mips);
println!("{{ \"type\": \"bench\", \"name\": \"bench_program_alu_jit_to_native_mips\", \"median\": {:?}, \"deviation\": 0 }}", mips);
}
#[bench]
@@ -220,125 +200,102 @@ fn bench_program_execute_noop(bencher: &mut Bencher) {
#[bench]
fn bench_create_vm(bencher: &mut Bencher) {
const BUDGET: u64 = 200_000;
let loader_id = bpf_loader::id();
let accounts = [RefCell::new(AccountSharedData::new(
1,
10000001,
&solana_sdk::pubkey::new_rand(),
))];
let keys = [solana_sdk::pubkey::new_rand()];
let keyed_accounts: Vec<_> = keys
.iter()
.zip(&accounts)
.map(|(key, account)| solana_sdk::keyed_account::KeyedAccount::new(&key, false, &account))
.collect();
let instruction_data = vec![0u8];
let mut invoke_context = MockInvokeContext::new(&loader_id, keyed_accounts);
invoke_context.compute_meter = Rc::new(RefCell::new(MockComputeMeter { remaining: BUDGET }));
// Serialize account data
let keyed_accounts = invoke_context.get_keyed_accounts().unwrap();
let (mut serialized, account_lengths) = serialize_parameters(
&loader_id,
&solana_sdk::pubkey::new_rand(),
keyed_accounts,
&instruction_data,
)
.unwrap();
let elf = load_elf("noop").unwrap();
let executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&elf,
None,
Config::default(),
register_syscalls(&mut invoke_context).unwrap(),
)
.unwrap();
let loader_id = bpf_loader::id();
with_mock_invoke_context(loader_id, 10000001, |invoke_context| {
const BUDGET: u64 = 200_000;
let compute_meter = invoke_context.get_compute_meter();
{
let mut compute_meter = compute_meter.borrow_mut();
let to_consume = compute_meter.get_remaining() - BUDGET;
compute_meter.consume(to_consume).unwrap();
}
bencher.iter(|| {
let _ = create_vm(
&loader_id,
executable.as_ref(),
serialized.as_slice_mut(),
&mut invoke_context,
&account_lengths,
// Serialize account data
let keyed_accounts = invoke_context.get_keyed_accounts().unwrap();
let (mut serialized, account_lengths) = serialize_parameters(
&keyed_accounts[0].unsigned_key(),
&keyed_accounts[1].unsigned_key(),
&keyed_accounts[2..],
&[],
)
.unwrap();
let executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&elf,
None,
Config::default(),
register_syscalls(invoke_context).unwrap(),
)
.unwrap();
bencher.iter(|| {
let _ = create_vm(
&loader_id,
executable.as_ref(),
serialized.as_slice_mut(),
invoke_context,
&account_lengths,
)
.unwrap();
});
});
}
#[bench]
fn bench_instruction_count_tuner(_bencher: &mut Bencher) {
const BUDGET: u64 = 200_000;
let loader_id = bpf_loader::id();
let program_id = solana_sdk::pubkey::new_rand();
let accounts = [
(
program_id,
RefCell::new(AccountSharedData::new(0, 0, &loader_id)),
),
(
solana_sdk::pubkey::new_rand(),
RefCell::new(AccountSharedData::new(
1,
10000001,
&solana_sdk::pubkey::new_rand(),
)),
),
];
let keyed_accounts: Vec<_> = accounts
.iter()
.map(|(key, account)| solana_sdk::keyed_account::KeyedAccount::new(&key, false, &account))
.collect();
let instruction_data = vec![0u8];
let mut invoke_context = MockInvokeContext::new(&program_id, keyed_accounts);
invoke_context.compute_meter = Rc::new(RefCell::new(MockComputeMeter { remaining: BUDGET }));
// Serialize account data
let (mut serialized, account_lengths) = serialize_parameters(
&loader_id,
&program_id,
&invoke_context.get_keyed_accounts().unwrap()[1..],
&instruction_data,
)
.unwrap();
let elf = load_elf("tuner").unwrap();
let executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&elf,
None,
Config::default(),
register_syscalls(&mut invoke_context).unwrap(),
)
.unwrap();
let compute_meter = invoke_context.get_compute_meter();
let mut instruction_meter = ThisInstructionMeter { compute_meter };
let mut vm = create_vm(
&loader_id,
executable.as_ref(),
serialized.as_slice_mut(),
&mut invoke_context,
&account_lengths,
)
.unwrap();
let loader_id = bpf_loader::id();
with_mock_invoke_context(loader_id, 10000001, |invoke_context| {
const BUDGET: u64 = 200_000;
let compute_meter = invoke_context.get_compute_meter();
{
let mut compute_meter = compute_meter.borrow_mut();
let to_consume = compute_meter.get_remaining() - BUDGET;
compute_meter.consume(to_consume).unwrap();
}
let mut measure = Measure::start("tune");
let _ = vm.execute_program_interpreted(&mut instruction_meter);
measure.stop();
// Serialize account data
let keyed_accounts = invoke_context.get_keyed_accounts().unwrap();
let (mut serialized, account_lengths) = serialize_parameters(
&keyed_accounts[0].unsigned_key(),
&keyed_accounts[1].unsigned_key(),
&keyed_accounts[2..],
&[],
)
.unwrap();
assert_eq!(
0,
instruction_meter.get_remaining(),
"Tuner must consume the whole budget"
);
println!(
"{:?} compute units took {:?} us ({:?} instructions)",
BUDGET - instruction_meter.get_remaining(),
measure.as_us(),
vm.get_total_instruction_count(),
);
let executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&elf,
None,
Config::default(),
register_syscalls(invoke_context).unwrap(),
)
.unwrap();
let mut instruction_meter = ThisInstructionMeter { compute_meter };
let mut vm = create_vm(
&loader_id,
executable.as_ref(),
serialized.as_slice_mut(),
invoke_context,
&account_lengths,
)
.unwrap();
let mut measure = Measure::start("tune");
let _ = vm.execute_program_interpreted(&mut instruction_meter);
measure.stop();
assert_eq!(
0,
instruction_meter.get_remaining(),
"Tuner must consume the whole budget"
);
println!(
"{:?} compute units took {:?} us ({:?} instructions)",
BUDGET - instruction_meter.get_remaining(),
measure.as_us(),
vm.get_total_instruction_count(),
);
});
}

View File

@@ -31,6 +31,7 @@ use solana_runtime::{
upgrade_program,
},
};
use solana_program_runtime::invoke_context::with_mock_invoke_context;
use solana_sdk::{
account::{AccountSharedData, ReadableAccount},
account_utils::StateMut,
@@ -40,10 +41,9 @@ use solana_sdk::{
compute_budget::{ComputeBudget, ComputeBudgetInstruction},
entrypoint::{MAX_PERMITTED_DATA_INCREASE, SUCCESS},
instruction::{AccountMeta, CompiledInstruction, Instruction, InstructionError},
keyed_account::KeyedAccount,
loader_instruction,
message::{Message, SanitizedMessage},
process_instruction::{InvokeContext, MockInvokeContext},
process_instruction::InvokeContext,
pubkey::Pubkey,
signature::{keypair_from_seed, Keypair, Signer},
system_instruction::{self, MAX_PERMITTED_DATA_LENGTH},
@@ -56,8 +56,8 @@ use solana_transaction_status::{
TransactionStatusMeta, TransactionWithStatusMeta, UiTransactionEncoding,
};
use std::{
cell::RefCell, collections::HashMap, convert::TryFrom, env, fs::File, io::Read, path::PathBuf,
str::FromStr, sync::Arc,
collections::HashMap, convert::TryFrom, env, fs::File, io::Read, path::PathBuf, str::FromStr,
sync::Arc,
};
/// BPF program file extension
@@ -190,109 +190,103 @@ fn upgrade_bpf_program(
);
}
fn run_program(
name: &str,
loader_id: &Pubkey,
program_id: &Pubkey,
parameter_accounts: Vec<KeyedAccount>,
instruction_data: &[u8],
) -> Result<u64, InstructionError> {
fn run_program(name: &str) -> u64 {
let mut file = File::open(create_bpf_path(name)).unwrap();
let mut data = vec![];
file.read_to_end(&mut data).unwrap();
let mut invoke_context = MockInvokeContext::new(&program_id, parameter_accounts);
let (parameter_bytes, account_lengths) = serialize_parameters(
&loader_id,
program_id,
&invoke_context.get_keyed_accounts().unwrap()[1..],
&instruction_data,
)
.unwrap();
let compute_meter = invoke_context.get_compute_meter();
let mut instruction_meter = ThisInstructionMeter { compute_meter };
let config = Config {
enable_instruction_tracing: true,
..Config::default()
};
let mut executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&data,
None,
config,
register_syscalls(&mut invoke_context).unwrap(),
)
.unwrap();
executable.jit_compile().unwrap();
let mut instruction_count = 0;
let mut tracer = None;
for i in 0..2 {
let mut parameter_bytes = parameter_bytes.clone();
{
invoke_context.set_return_data(Vec::new()).unwrap();
let mut vm = create_vm(
&loader_id,
executable.as_ref(),
parameter_bytes.as_slice_mut(),
&mut invoke_context,
&account_lengths,
)
.unwrap();
let result = if i == 0 {
vm.execute_program_interpreted(&mut instruction_meter)
} else {
vm.execute_program_jit(&mut instruction_meter)
};
assert_eq!(SUCCESS, result.unwrap());
if i == 1 {
assert_eq!(instruction_count, vm.get_total_instruction_count());
}
instruction_count = vm.get_total_instruction_count();
if config.enable_instruction_tracing {
if i == 1 {
if !Tracer::compare(tracer.as_ref().unwrap(), vm.get_tracer()) {
let analysis = Analysis::from_executable(executable.as_ref());
let stdout = std::io::stdout();
println!("TRACE (interpreted):");
tracer
.as_ref()
.unwrap()
.write(&mut stdout.lock(), &analysis)
.unwrap();
println!("TRACE (jit):");
vm.get_tracer()
.write(&mut stdout.lock(), &analysis)
.unwrap();
assert!(false);
} else if log_enabled!(Trace) {
let analysis = Analysis::from_executable(executable.as_ref());
let mut trace_buffer = Vec::<u8>::new();
tracer
.as_ref()
.unwrap()
.write(&mut trace_buffer, &analysis)
.unwrap();
let trace_string = String::from_utf8(trace_buffer).unwrap();
trace!("BPF Program Instruction Trace:\n{}", trace_string);
}
}
tracer = Some(vm.get_tracer().clone());
}
}
let parameter_accounts = invoke_context.get_keyed_accounts().unwrap();
deserialize_parameters(
&loader_id,
parameter_accounts,
parameter_bytes.as_slice(),
&account_lengths,
true,
let loader_id = bpf_loader::id();
with_mock_invoke_context(loader_id, 0, |invoke_context| {
let keyed_accounts = invoke_context.get_keyed_accounts().unwrap();
let (parameter_bytes, account_lengths) = serialize_parameters(
&keyed_accounts[0].unsigned_key(),
&keyed_accounts[1].unsigned_key(),
&keyed_accounts[2..],
&[],
)
.unwrap();
}
Ok(instruction_count)
let compute_meter = invoke_context.get_compute_meter();
let mut instruction_meter = ThisInstructionMeter { compute_meter };
let config = Config {
enable_instruction_tracing: true,
..Config::default()
};
let mut executable = <dyn Executable<BpfError, ThisInstructionMeter>>::from_elf(
&data,
None,
config,
register_syscalls(invoke_context).unwrap(),
)
.unwrap();
executable.jit_compile().unwrap();
let mut instruction_count = 0;
let mut tracer = None;
for i in 0..2 {
invoke_context.set_return_data(Vec::new()).unwrap();
let mut parameter_bytes = parameter_bytes.clone();
{
let mut vm = create_vm(
&loader_id,
executable.as_ref(),
parameter_bytes.as_slice_mut(),
invoke_context,
&account_lengths,
)
.unwrap();
let result = if i == 0 {
vm.execute_program_interpreted(&mut instruction_meter)
} else {
vm.execute_program_jit(&mut instruction_meter)
};
assert_eq!(SUCCESS, result.unwrap());
if i == 1 {
assert_eq!(instruction_count, vm.get_total_instruction_count());
}
instruction_count = vm.get_total_instruction_count();
if config.enable_instruction_tracing {
if i == 1 {
if !Tracer::compare(tracer.as_ref().unwrap(), vm.get_tracer()) {
let analysis = Analysis::from_executable(executable.as_ref());
let stdout = std::io::stdout();
println!("TRACE (interpreted):");
tracer
.as_ref()
.unwrap()
.write(&mut stdout.lock(), &analysis)
.unwrap();
println!("TRACE (jit):");
vm.get_tracer()
.write(&mut stdout.lock(), &analysis)
.unwrap();
assert!(false);
} else if log_enabled!(Trace) {
let analysis = Analysis::from_executable(executable.as_ref());
let mut trace_buffer = Vec::<u8>::new();
tracer
.as_ref()
.unwrap()
.write(&mut trace_buffer, &analysis)
.unwrap();
let trace_string = String::from_utf8(trace_buffer).unwrap();
trace!("BPF Program Instruction Trace:\n{}", trace_string);
}
}
tracer = Some(vm.get_tracer().clone());
}
}
let keyed_accounts = invoke_context.get_keyed_accounts().unwrap();
deserialize_parameters(
&loader_id,
&keyed_accounts[2..],
parameter_bytes.as_slice(),
&account_lengths,
true,
)
.unwrap();
}
instruction_count
})
}
fn process_transaction_and_record_inner(
@@ -1377,8 +1371,8 @@ fn assert_instruction_count() {
("noop++", 5),
("relative_call", 26),
("return_data", 980),
("sanity", 1246),
("sanity++", 1250),
("sanity", 1248),
("sanity++", 1252),
("secp256k1_recover", 25383),
("sha", 1328),
("struct_pass", 108),
@@ -1390,17 +1384,17 @@ fn assert_instruction_count() {
programs.extend_from_slice(&[
("solana_bpf_rust_128bit", 584),
("solana_bpf_rust_alloc", 7388),
("solana_bpf_rust_custom_heap", 535),
("solana_bpf_rust_custom_heap", 536),
("solana_bpf_rust_dep_crate", 47),
("solana_bpf_rust_external_spend", 506),
("solana_bpf_rust_external_spend", 507),
("solana_bpf_rust_iter", 824),
("solana_bpf_rust_many_args", 941),
("solana_bpf_rust_mem", 3085),
("solana_bpf_rust_mem", 3086),
("solana_bpf_rust_membuiltins", 3976),
("solana_bpf_rust_noop", 480),
("solana_bpf_rust_noop", 481),
("solana_bpf_rust_param_passing", 146),
("solana_bpf_rust_rand", 487),
("solana_bpf_rust_sanity", 8454),
("solana_bpf_rust_rand", 488),
("solana_bpf_rust_sanity", 8455),
("solana_bpf_rust_secp256k1_recover", 25216),
("solana_bpf_rust_sha", 30692),
]);
@@ -1409,17 +1403,7 @@ fn assert_instruction_count() {
let mut passed = true;
println!("\n {:36} expected actual diff", "BPF program");
for program in programs.iter() {
let loader_id = bpf_loader::id();
let program_id = Pubkey::new_unique();
let key = Pubkey::new_unique();
let mut program_account = RefCell::new(AccountSharedData::new(0, 0, &loader_id));
let mut account = RefCell::new(AccountSharedData::default());
let parameter_accounts = vec![
KeyedAccount::new(&program_id, false, &mut program_account),
KeyedAccount::new(&key, false, &mut account),
];
let count =
run_program(program.0, &loader_id, &program_id, parameter_accounts, &[]).unwrap();
let count = run_program(program.0);
let diff: i64 = count as i64 - program.1 as i64;
println!(
" {:36} {:8} {:6} {:+5} ({:+3.0}%)",