From db3fb3a27cfd5c60c371ad7b567e3e669a2174fb Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Wed, 22 Aug 2018 08:57:07 -0600 Subject: [PATCH] Boot criterion (#1032) * Revert benchmarks back to libtest Criterion has too many dependencies, it's execution as slower, and we didn't see the kind of precision we had hoped for to use it to block CI builds. * Ignore benchmarks that take more than a few milliseconds per iteration * Revert "Ignore benchmarks that take more than a few milliseconds per iteration" This reverts commit b87cdf6ef4a2549ec09e282014391de695473699. * Don't run benchmarks in CI They are already built in the nightly build. Executing them in CI doesn't add much value until the results are precise enough to act on. --- Cargo.toml | 8 -------- benches/bank.rs | 37 ++++++++++--------------------------- benches/banking_stage.rs | 27 +++++---------------------- benches/ledger.rs | 20 ++++---------------- benches/signature.rs | 20 ++++---------------- benches/sigverify.rs | 20 ++++---------------- ci/buildkite.yml | 10 +++++----- ci/test-bench.sh | 4 ++-- 8 files changed, 34 insertions(+), 112 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ae911f0acb..824d9b878c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,25 +94,17 @@ tokio = "0.1" tokio-codec = "0.1" untrusted = "0.6.2" -[dev-dependencies] -criterion = "0.2" - [[bench]] name = "bank" -harness = false [[bench]] name = "banking_stage" -harness = false [[bench]] name = "ledger" -harness = false [[bench]] name = "signature" -harness = false [[bench]] name = "sigverify" -harness = false diff --git a/benches/bank.rs b/benches/bank.rs index 08f00d4a82..93a713929f 100644 --- a/benches/bank.rs +++ b/benches/bank.rs @@ -1,18 +1,19 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate bincode; extern crate rayon; extern crate solana; +extern crate test; use bincode::serialize; -use criterion::{Bencher, Criterion}; use rayon::prelude::*; use solana::bank::*; use solana::hash::hash; use solana::mint::Mint; use solana::signature::{Keypair, KeypairUtil}; use solana::transaction::Transaction; +use test::Bencher; +#[bench] fn bench_process_transaction(bencher: &mut Bencher) { let mint = Mint::new(100_000_000); let bank = Bank::new(&mint); @@ -39,28 +40,10 @@ fn bench_process_transaction(bencher: &mut Bencher) { }) .collect(); - bencher.iter_with_setup( - || { - // Since benchmarker runs this multiple times, we need to clear the signatures. - bank.clear_signatures(); - transactions.clone() - }, - |transactions| { - let results = bank.process_transactions(transactions); - assert!(results.iter().all(Result::is_ok)); - }, - ) + bencher.iter(|| { + // Since benchmarker runs this multiple times, we need to clear the signatures. + bank.clear_signatures(); + let results = bank.process_transactions(transactions.clone()); + assert!(results.iter().all(Result::is_ok)); + }) } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_process_transaction", |bencher| { - bench_process_transaction(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/banking_stage.rs b/benches/banking_stage.rs index 43a2dd3a87..f1aa6dc116 100644 --- a/benches/banking_stage.rs +++ b/benches/banking_stage.rs @@ -1,10 +1,9 @@ +#![feature(test)] extern crate bincode; -#[macro_use] -extern crate criterion; extern crate rayon; extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use rayon::prelude::*; use solana::bank::Bank; use solana::banking_stage::BankingStage; @@ -16,6 +15,7 @@ use solana::transaction::Transaction; use std::iter; use std::sync::mpsc::{channel, Receiver}; use std::sync::Arc; +use test::Bencher; // use self::test::Bencher; // use bank::{Bank, MAX_ENTRY_IDS}; @@ -95,6 +95,7 @@ fn check_txs(receiver: &Receiver, ref_tx_count: usize) { assert_eq!(total, ref_tx_count); } +#[bench] fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { let tx = 10_000_usize; let mint_total = 1_000_000_000_000; @@ -145,7 +146,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { }) .collect(); - let verified_setup_len = verified_setup.len(); verified_sender.send(verified_setup).unwrap(); BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler) .unwrap(); @@ -160,7 +160,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { }) .collect(); - let verified_len = verified.len(); verified_sender.send(verified).unwrap(); BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler) .unwrap(); @@ -169,6 +168,7 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { }); } +#[bench] fn bench_banking_stage_single_from(bencher: &mut Bencher) { let tx = 10_000_usize; let mint = Mint::new(1_000_000_000_000); @@ -203,7 +203,6 @@ fn bench_banking_stage_single_from(bencher: &mut Bencher) { (x, iter::repeat(1).take(len).collect()) }) .collect(); - let verified_len = verified.len(); verified_sender.send(verified).unwrap(); BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler) .unwrap(); @@ -211,19 +210,3 @@ fn bench_banking_stage_single_from(bencher: &mut Bencher) { check_txs(&signal_receiver, tx); }); } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_banking_stage_multi_accounts", |bencher| { - bench_banking_stage_multi_accounts(bencher); - }); - criterion.bench_function("bench_process_stage_single_from", |bencher| { - bench_banking_stage_single_from(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/ledger.rs b/benches/ledger.rs index f0db373db0..c7fe169b25 100644 --- a/benches/ledger.rs +++ b/benches/ledger.rs @@ -1,15 +1,16 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use solana::hash::{hash, Hash}; use solana::ledger::{next_entries, reconstruct_entries_from_blobs, Block}; use solana::packet::BlobRecycler; use solana::signature::{Keypair, KeypairUtil}; use solana::transaction::Transaction; use std::collections::VecDeque; +use test::Bencher; +#[bench] fn bench_block_to_blobs_to_block(bencher: &mut Bencher) { let zero = Hash::default(); let one = hash(&zero.as_ref()); @@ -25,16 +26,3 @@ fn bench_block_to_blobs_to_block(bencher: &mut Bencher) { assert_eq!(reconstruct_entries_from_blobs(blob_q).unwrap(), entries); }); } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_block_to_blobs_to_block", |bencher| { - bench_block_to_blobs_to_block(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/signature.rs b/benches/signature.rs index 651d2766be..3d8646d54f 100644 --- a/benches/signature.rs +++ b/benches/signature.rs @@ -1,24 +1,12 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use solana::signature::GenKeys; +use test::Bencher; +#[bench] fn bench_gen_keys(b: &mut Bencher) { let mut rnd = GenKeys::new([0u8; 32]); b.iter(|| rnd.gen_n_keypairs(1000)); } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_gen_keys", |bencher| { - bench_gen_keys(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/sigverify.rs b/benches/sigverify.rs index b1cf3f7466..013f891232 100644 --- a/benches/sigverify.rs +++ b/benches/sigverify.rs @@ -1,14 +1,15 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate bincode; extern crate rayon; extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use solana::packet::{to_packets, PacketRecycler}; use solana::sigverify; use solana::transaction::test_tx; +use test::Bencher; +#[bench] fn bench_sigverify(bencher: &mut Bencher) { let tx = test_tx(); @@ -21,16 +22,3 @@ fn bench_sigverify(bencher: &mut Bencher) { let _ans = sigverify::ed25519_verify(&batches); }) } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_sigverify", |bencher| { - bench_sigverify(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/ci/buildkite.yml b/ci/buildkite.yml index ada02418dd..94167fb387 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -4,11 +4,11 @@ steps: env: CARGO_TARGET_CACHE_NAME: "stable" timeout_in_minutes: 30 - - command: "ci/docker-run.sh solanalabs/rust ci/test-bench.sh" - name: "bench [public]" - env: - CARGO_TARGET_CACHE_NAME: "stable" - timeout_in_minutes: 30 + # - command: "ci/docker-run.sh solanalabs/rust-nightly ci/test-bench.sh" + # name: "bench [public]" + # env: + # CARGO_TARGET_CACHE_NAME: "nightly" + # timeout_in_minutes: 30 - command: "ci/shellcheck.sh" name: "shellcheck [public]" timeout_in_minutes: 20 diff --git a/ci/test-bench.sh b/ci/test-bench.sh index 4541b51dda..0311667206 100755 --- a/ci/test-bench.sh +++ b/ci/test-bench.sh @@ -2,7 +2,7 @@ cd "$(dirname "$0")/.." -ci/version-check.sh stable +ci/version-check.sh nightly export RUST_BACKTRACE=1 _() { @@ -10,4 +10,4 @@ _() { "$@" } -_ cargo bench --verbose +_ cargo bench --features=unstable --verbose