Boot criterion (#1032)

* Revert benchmarks back to libtest

Criterion has too many dependencies, it's execution as slower, and
we didn't see the kind of precision we had hoped for to use it to
block CI builds.

* Ignore benchmarks that take more than a few milliseconds per iteration

* Revert "Ignore benchmarks that take more than a few milliseconds per iteration"

This reverts commit b87cdf6ef4.

* Don't run benchmarks in CI

They are already built in the nightly build. Executing them in CI
doesn't add much value until the results are precise enough to act
on.

Cargo.toml

@@ -94,25 +94,17 @@ tokio = "0.1"
 tokio-codec = "0.1"
 untrusted = "0.6.2"
 
-[dev-dependencies]
-criterion = "0.2"
-
 [[bench]]
 name = "bank"
-harness = false
 
 [[bench]]
 name = "banking_stage"
-harness = false
 
 [[bench]]
 name = "ledger"
-harness = false
 
 [[bench]]
 name = "signature"
-harness = false
 
 [[bench]]
 name = "sigverify"
-harness = false

benches/bank.rs

@@ -1,18 +1,19 @@
-#[macro_use]
+#![feature(test)]
-extern crate criterion;
 extern crate bincode;
 extern crate rayon;
 extern crate solana;
+extern crate test;
 
 use bincode::serialize;
-use criterion::{Bencher, Criterion};
 use rayon::prelude::*;
 use solana::bank::*;
 use solana::hash::hash;
 use solana::mint::Mint;
 use solana::signature::{Keypair, KeypairUtil};
 use solana::transaction::Transaction;
+use test::Bencher;
 
+#[bench]
 fn bench_process_transaction(bencher: &mut Bencher) {
     let mint = Mint::new(100_000_000);
     let bank = Bank::new(&mint);
@@ -39,28 +40,10 @@ fn bench_process_transaction(bencher: &mut Bencher) {
         })
         .collect();
 
-    bencher.iter_with_setup(
+    bencher.iter(|| {
-        || {
+        // Since benchmarker runs this multiple times, we need to clear the signatures.
-            // Since benchmarker runs this multiple times, we need to clear the signatures.
+        bank.clear_signatures();
-            bank.clear_signatures();
+        let results = bank.process_transactions(transactions.clone());
-            transactions.clone()
+        assert!(results.iter().all(Result::is_ok));
-        },
+    })
-        |transactions| {
-            let results = bank.process_transactions(transactions);
-            assert!(results.iter().all(Result::is_ok));
-        },
-    )
 }
-
-fn bench(criterion: &mut Criterion) {
-    criterion.bench_function("bench_process_transaction", |bencher| {
-        bench_process_transaction(bencher);
-    });
-}
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default().sample_size(2);
-    targets = bench
-);
-criterion_main!(benches);

benches/banking_stage.rs

@@ -1,10 +1,9 @@
+#![feature(test)]
 extern crate bincode;
-#[macro_use]
-extern crate criterion;
 extern crate rayon;
 extern crate solana;
+extern crate test;
 
-use criterion::{Bencher, Criterion};
 use rayon::prelude::*;
 use solana::bank::Bank;
 use solana::banking_stage::BankingStage;
@@ -16,6 +15,7 @@ use solana::transaction::Transaction;
 use std::iter;
 use std::sync::mpsc::{channel, Receiver};
 use std::sync::Arc;
+use test::Bencher;
 
 // use self::test::Bencher;
 // use bank::{Bank, MAX_ENTRY_IDS};
@@ -95,6 +95,7 @@ fn check_txs(receiver: &Receiver<Signal>, ref_tx_count: usize) {
     assert_eq!(total, ref_tx_count);
 }
 
+#[bench]
 fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
     let tx = 10_000_usize;
     let mint_total = 1_000_000_000_000;
@@ -145,7 +146,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
                 })
                 .collect();
 
-        let verified_setup_len = verified_setup.len();
         verified_sender.send(verified_setup).unwrap();
         BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
             .unwrap();
@@ -160,7 +160,6 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
             })
             .collect();
 
-        let verified_len = verified.len();
         verified_sender.send(verified).unwrap();
         BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
             .unwrap();
@@ -169,6 +168,7 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
     });
 }
 
+#[bench]
 fn bench_banking_stage_single_from(bencher: &mut Bencher) {
     let tx = 10_000_usize;
     let mint = Mint::new(1_000_000_000_000);
@@ -203,7 +203,6 @@ fn bench_banking_stage_single_from(bencher: &mut Bencher) {
                 (x, iter::repeat(1).take(len).collect())
             })
             .collect();
-        let verified_len = verified.len();
         verified_sender.send(verified).unwrap();
         BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
             .unwrap();
@@ -211,19 +210,3 @@ fn bench_banking_stage_single_from(bencher: &mut Bencher) {
         check_txs(&signal_receiver, tx);
     });
 }
-
-fn bench(criterion: &mut Criterion) {
-    criterion.bench_function("bench_banking_stage_multi_accounts", |bencher| {
-        bench_banking_stage_multi_accounts(bencher);
-    });
-    criterion.bench_function("bench_process_stage_single_from", |bencher| {
-        bench_banking_stage_single_from(bencher);
-    });
-}
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default().sample_size(2);
-    targets = bench
-);
-criterion_main!(benches);

benches/ledger.rs

@@ -1,15 +1,16 @@
-#[macro_use]
+#![feature(test)]
-extern crate criterion;
 extern crate solana;
+extern crate test;
 
-use criterion::{Bencher, Criterion};
 use solana::hash::{hash, Hash};
 use solana::ledger::{next_entries, reconstruct_entries_from_blobs, Block};
 use solana::packet::BlobRecycler;
 use solana::signature::{Keypair, KeypairUtil};
 use solana::transaction::Transaction;
 use std::collections::VecDeque;
+use test::Bencher;
 
+#[bench]
 fn bench_block_to_blobs_to_block(bencher: &mut Bencher) {
     let zero = Hash::default();
     let one = hash(&zero.as_ref());
@@ -25,16 +26,3 @@ fn bench_block_to_blobs_to_block(bencher: &mut Bencher) {
         assert_eq!(reconstruct_entries_from_blobs(blob_q).unwrap(), entries);
     });
 }
-
-fn bench(criterion: &mut Criterion) {
-    criterion.bench_function("bench_block_to_blobs_to_block", |bencher| {
-        bench_block_to_blobs_to_block(bencher);
-    });
-}
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default().sample_size(2);
-    targets = bench
-);
-criterion_main!(benches);

benches/signature.rs

@@ -1,24 +1,12 @@
-#[macro_use]
+#![feature(test)]
-extern crate criterion;
 extern crate solana;
+extern crate test;
 
-use criterion::{Bencher, Criterion};
 use solana::signature::GenKeys;
+use test::Bencher;
 
+#[bench]
 fn bench_gen_keys(b: &mut Bencher) {
     let mut rnd = GenKeys::new([0u8; 32]);
     b.iter(|| rnd.gen_n_keypairs(1000));
 }
-
-fn bench(criterion: &mut Criterion) {
-    criterion.bench_function("bench_gen_keys", |bencher| {
-        bench_gen_keys(bencher);
-    });
-}
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default().sample_size(2);
-    targets = bench
-);
-criterion_main!(benches);

benches/sigverify.rs

@@ -1,14 +1,15 @@
-#[macro_use]
+#![feature(test)]
-extern crate criterion;
 extern crate bincode;
 extern crate rayon;
 extern crate solana;
+extern crate test;
 
-use criterion::{Bencher, Criterion};
 use solana::packet::{to_packets, PacketRecycler};
 use solana::sigverify;
 use solana::transaction::test_tx;
+use test::Bencher;
 
+#[bench]
 fn bench_sigverify(bencher: &mut Bencher) {
     let tx = test_tx();
 
@@ -21,16 +22,3 @@ fn bench_sigverify(bencher: &mut Bencher) {
         let _ans = sigverify::ed25519_verify(&batches);
     })
 }
-
-fn bench(criterion: &mut Criterion) {
-    criterion.bench_function("bench_sigverify", |bencher| {
-        bench_sigverify(bencher);
-    });
-}
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default().sample_size(2);
-    targets = bench
-);
-criterion_main!(benches);

ci/buildkite.yml

@@ -4,11 +4,11 @@ steps:
     env:
       CARGO_TARGET_CACHE_NAME: "stable"
     timeout_in_minutes: 30
-  - command: "ci/docker-run.sh solanalabs/rust ci/test-bench.sh"
+    #  - command: "ci/docker-run.sh solanalabs/rust-nightly ci/test-bench.sh"
-    name: "bench [public]"
+    #    name: "bench [public]"
-    env:
+    #    env:
-      CARGO_TARGET_CACHE_NAME: "stable"
+    #      CARGO_TARGET_CACHE_NAME: "nightly"
-    timeout_in_minutes: 30
+    #    timeout_in_minutes: 30
   - command: "ci/shellcheck.sh"
     name: "shellcheck [public]"
     timeout_in_minutes: 20

ci/test-bench.sh

@@ -2,7 +2,7 @@
 
 cd "$(dirname "$0")/.."
 
-ci/version-check.sh stable
+ci/version-check.sh nightly
 export RUST_BACKTRACE=1
 
 _() {
@@ -10,4 +10,4 @@ _() {
   "$@"
 }
 
-_ cargo bench --verbose
+_ cargo bench --features=unstable --verbose