From 95543c615aaef3ac5c858ef86c4bb265e341112f Mon Sep 17 00:00:00 2001 From: Greg Fitzgerald Date: Tue, 21 Aug 2018 15:26:05 -0600 Subject: [PATCH] Revert benchmarks back to libtest Criterion has too many dependencies, it's execution as slower, and we didn't see the kind of precision we had hoped for to use it to block CI builds. --- Cargo.toml | 8 -------- benches/bank.rs | 37 ++++++++++--------------------------- benches/banking_stage.rs | 27 +++++---------------------- benches/ledger.rs | 20 ++++---------------- benches/signature.rs | 20 ++++---------------- benches/sigverify.rs | 20 ++++---------------- ci/buildkite.yml | 4 ++-- ci/test-bench.sh | 4 ++-- 8 files changed, 31 insertions(+), 109 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ae911f0acb..824d9b878c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,25 +94,17 @@ tokio = "0.1" tokio-codec = "0.1" untrusted = "0.6.2" -[dev-dependencies] -criterion = "0.2" - [[bench]] name = "bank" -harness = false [[bench]] name = "banking_stage" -harness = false [[bench]] name = "ledger" -harness = false [[bench]] name = "signature" -harness = false [[bench]] name = "sigverify" -harness = false diff --git a/benches/bank.rs b/benches/bank.rs index 08f00d4a82..93a713929f 100644 --- a/benches/bank.rs +++ b/benches/bank.rs @@ -1,18 +1,19 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate bincode; extern crate rayon; extern crate solana; +extern crate test; use bincode::serialize; -use criterion::{Bencher, Criterion}; use rayon::prelude::*; use solana::bank::*; use solana::hash::hash; use solana::mint::Mint; use solana::signature::{Keypair, KeypairUtil}; use solana::transaction::Transaction; +use test::Bencher; +#[bench] fn bench_process_transaction(bencher: &mut Bencher) { let mint = Mint::new(100_000_000); let bank = Bank::new(&mint); @@ -39,28 +40,10 @@ fn bench_process_transaction(bencher: &mut Bencher) { }) .collect(); - bencher.iter_with_setup( - || { - // Since benchmarker runs this multiple times, we need to clear the signatures. - bank.clear_signatures(); - transactions.clone() - }, - |transactions| { - let results = bank.process_transactions(transactions); - assert!(results.iter().all(Result::is_ok)); - }, - ) + bencher.iter(|| { + // Since benchmarker runs this multiple times, we need to clear the signatures. + bank.clear_signatures(); + let results = bank.process_transactions(transactions.clone()); + assert!(results.iter().all(Result::is_ok)); + }) } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_process_transaction", |bencher| { - bench_process_transaction(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/banking_stage.rs b/benches/banking_stage.rs index 43a2dd3a87..f1aa6dc116 100644 --- a/benches/banking_stage.rs +++ b/benches/banking_stage.rs @@ -1,10 +1,9 @@ +#![feature(test)] extern crate bincode; -#[macro_use] -extern crate criterion; extern crate rayon; extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use rayon::prelude::*; use solana::bank::Bank; use solana::banking_stage::BankingStage; @@ -16,6 +15,7 @@ use solana::transaction::Transaction; use std::iter; use std::sync::mpsc::{channel, Receiver}; use std::sync::Arc; +use test::Bencher; // use self::test::Bencher; // use bank::{Bank, MAX_ENTRY_IDS}; @@ -95,6 +95,7 @@ fn check_txs(receiver: &Receiver, ref_tx_count: usize) { assert_eq!(total, ref_tx_count); } +#[bench] fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { let tx = 10_000_usize; let mint_total = 1_000_000_000_000; @@ -145,7 +146,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { }) .collect(); - let verified_setup_len = verified_setup.len(); verified_sender.send(verified_setup).unwrap(); BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler) .unwrap(); @@ -160,7 +160,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { }) .collect(); - let verified_len = verified.len(); verified_sender.send(verified).unwrap(); BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler) .unwrap(); @@ -169,6 +168,7 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) { }); } +#[bench] fn bench_banking_stage_single_from(bencher: &mut Bencher) { let tx = 10_000_usize; let mint = Mint::new(1_000_000_000_000); @@ -203,7 +203,6 @@ fn bench_banking_stage_single_from(bencher: &mut Bencher) { (x, iter::repeat(1).take(len).collect()) }) .collect(); - let verified_len = verified.len(); verified_sender.send(verified).unwrap(); BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler) .unwrap(); @@ -211,19 +210,3 @@ fn bench_banking_stage_single_from(bencher: &mut Bencher) { check_txs(&signal_receiver, tx); }); } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_banking_stage_multi_accounts", |bencher| { - bench_banking_stage_multi_accounts(bencher); - }); - criterion.bench_function("bench_process_stage_single_from", |bencher| { - bench_banking_stage_single_from(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/ledger.rs b/benches/ledger.rs index f0db373db0..c7fe169b25 100644 --- a/benches/ledger.rs +++ b/benches/ledger.rs @@ -1,15 +1,16 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use solana::hash::{hash, Hash}; use solana::ledger::{next_entries, reconstruct_entries_from_blobs, Block}; use solana::packet::BlobRecycler; use solana::signature::{Keypair, KeypairUtil}; use solana::transaction::Transaction; use std::collections::VecDeque; +use test::Bencher; +#[bench] fn bench_block_to_blobs_to_block(bencher: &mut Bencher) { let zero = Hash::default(); let one = hash(&zero.as_ref()); @@ -25,16 +26,3 @@ fn bench_block_to_blobs_to_block(bencher: &mut Bencher) { assert_eq!(reconstruct_entries_from_blobs(blob_q).unwrap(), entries); }); } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_block_to_blobs_to_block", |bencher| { - bench_block_to_blobs_to_block(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/signature.rs b/benches/signature.rs index 651d2766be..3d8646d54f 100644 --- a/benches/signature.rs +++ b/benches/signature.rs @@ -1,24 +1,12 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use solana::signature::GenKeys; +use test::Bencher; +#[bench] fn bench_gen_keys(b: &mut Bencher) { let mut rnd = GenKeys::new([0u8; 32]); b.iter(|| rnd.gen_n_keypairs(1000)); } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_gen_keys", |bencher| { - bench_gen_keys(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/benches/sigverify.rs b/benches/sigverify.rs index b1cf3f7466..013f891232 100644 --- a/benches/sigverify.rs +++ b/benches/sigverify.rs @@ -1,14 +1,15 @@ -#[macro_use] -extern crate criterion; +#![feature(test)] extern crate bincode; extern crate rayon; extern crate solana; +extern crate test; -use criterion::{Bencher, Criterion}; use solana::packet::{to_packets, PacketRecycler}; use solana::sigverify; use solana::transaction::test_tx; +use test::Bencher; +#[bench] fn bench_sigverify(bencher: &mut Bencher) { let tx = test_tx(); @@ -21,16 +22,3 @@ fn bench_sigverify(bencher: &mut Bencher) { let _ans = sigverify::ed25519_verify(&batches); }) } - -fn bench(criterion: &mut Criterion) { - criterion.bench_function("bench_sigverify", |bencher| { - bench_sigverify(bencher); - }); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(2); - targets = bench -); -criterion_main!(benches); diff --git a/ci/buildkite.yml b/ci/buildkite.yml index ada02418dd..0e814c784f 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -4,10 +4,10 @@ steps: env: CARGO_TARGET_CACHE_NAME: "stable" timeout_in_minutes: 30 - - command: "ci/docker-run.sh solanalabs/rust ci/test-bench.sh" + - command: "ci/docker-run.sh solanalabs/rust-nightly ci/test-bench.sh" name: "bench [public]" env: - CARGO_TARGET_CACHE_NAME: "stable" + CARGO_TARGET_CACHE_NAME: "nightly" timeout_in_minutes: 30 - command: "ci/shellcheck.sh" name: "shellcheck [public]" diff --git a/ci/test-bench.sh b/ci/test-bench.sh index 4541b51dda..0311667206 100755 --- a/ci/test-bench.sh +++ b/ci/test-bench.sh @@ -2,7 +2,7 @@ cd "$(dirname "$0")/.." -ci/version-check.sh stable +ci/version-check.sh nightly export RUST_BACKTRACE=1 _() { @@ -10,4 +10,4 @@ _() { "$@" } -_ cargo bench --verbose +_ cargo bench --features=unstable --verbose