Track votes from gossip for optimistic confirmation (#11209)

* Add check in cluster_info_vote_listenere to see if optimstic conf was achieved
Add OptimisticConfirmationVerifier

* More fixes

* Fix merge conflicts

* Remove gossip notificatin

* Add dashboards

* Fix rebase

* Count switch votes as well toward optimistic conf

* rename

Co-authored-by: Carl <carl@solana.com>

core/src/consensus.rs

@@ -762,6 +762,7 @@ pub mod test {
                             &keypairs.node_keypair,
                             &keypairs.vote_keypair,
                             &keypairs.vote_keypair,
+                            None,
                         );
                         info!("voting {} {}", parent_bank.slot(), parent_bank.hash());
                         new_bank.process_transaction(&vote_tx).unwrap();

core/src/lib.rs

@@ -35,6 +35,7 @@ pub mod heaviest_subtree_fork_choice;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
 pub mod non_circulating_supply;
+pub mod optimistic_confirmation_verifier;
 pub mod poh_recorder;
 pub mod poh_service;
 pub mod progress_map;
@@ -66,6 +67,7 @@ pub mod tree_diff;
 pub mod tvu;
 pub mod validator;
 pub mod verified_vote_packets;
+pub mod vote_stake_tracker;
 pub mod weighted_shuffle;
 pub mod window_service;
 

core/src/optimistic_confirmation_verifier.rs

@@ -0,0 +1,340 @@
+use crate::cluster_info_vote_listener::VoteTracker;
+use solana_ledger::blockstore::Blockstore;
+use solana_runtime::bank::Bank;
+use solana_sdk::{clock::Slot, hash::Hash};
+use std::{collections::BTreeSet, time::Instant};
+
+pub struct OptimisticConfirmationVerifier {
+    snapshot_start_slot: Slot,
+    unchecked_slots: BTreeSet<(Slot, Hash)>,
+    last_optimistic_slot_ts: Instant,
+}
+
+impl OptimisticConfirmationVerifier {
+    pub fn new(snapshot_start_slot: Slot) -> Self {
+        Self {
+            snapshot_start_slot,
+            unchecked_slots: BTreeSet::default(),
+            last_optimistic_slot_ts: Instant::now(),
+        }
+    }
+
+    // Returns any optimistic slots that were not rooted
+    pub fn get_unrooted_optimistic_slots(
+        &mut self,
+        root_bank: &Bank,
+        blockstore: &Blockstore,
+    ) -> Vec<(Slot, Hash)> {
+        let root = root_bank.slot();
+        let root_ancestors = &root_bank.ancestors;
+        let mut slots_before_root = self
+            .unchecked_slots
+            .split_off(&((root + 1), Hash::default()));
+        // `slots_before_root` now contains all slots <= root
+        std::mem::swap(&mut slots_before_root, &mut self.unchecked_slots);
+        slots_before_root
+            .into_iter()
+            .filter(|(optimistic_slot, hash)| {
+                (*optimistic_slot == root && *hash != root_bank.hash())
+                    || (!root_ancestors.contains_key(&optimistic_slot) &&
+                    // In this second part of the `and`, we account for the possibility that
+                    // there was some other root `rootX` set in BankForks where:
+                    //
+                    // `root` > `rootX` > `optimistic_slot`
+                    //
+                    // in which case `root` may  not contain the ancestor information for
+                    // slots < `rootX`, so we also have to check if `optimistic_slot` was rooted
+                    // through blockstore.
+                    !blockstore.is_root(*optimistic_slot))
+            })
+            .collect()
+    }
+
+    pub fn add_new_optimistic_confirmed_slots(&mut self, new_optimistic_slots: Vec<(Slot, Hash)>) {
+        if new_optimistic_slots.is_empty() {
+            return;
+        }
+
+        datapoint_info!(
+            "optimistic_slot_elapsed",
+            (
+                "average_elapsed_ms",
+                self.last_optimistic_slot_ts.elapsed().as_millis() as i64,
+                i64
+            ),
+        );
+
+        // We don't have any information about ancestors before the snapshot root,
+        // so ignore those slots
+        for (new_optimistic_slot, hash) in new_optimistic_slots {
+            if new_optimistic_slot > self.snapshot_start_slot {
+                datapoint_info!("optimistic_slot", ("slot", new_optimistic_slot, i64),);
+                self.unchecked_slots.insert((new_optimistic_slot, hash));
+            }
+        }
+
+        self.last_optimistic_slot_ts = Instant::now();
+    }
+
+    pub fn log_unrooted_optimistic_slots(
+        root_bank: &Bank,
+        vote_tracker: &VoteTracker,
+        unrooted_optimistic_slots: &[(Slot, Hash)],
+    ) {
+        let root = root_bank.slot();
+        for (optimistic_slot, hash) in unrooted_optimistic_slots.iter() {
+            let epoch = root_bank.epoch_schedule().get_epoch(*optimistic_slot);
+            let epoch_stakes = root_bank.epoch_stakes(epoch);
+            let total_epoch_stake = epoch_stakes.map(|e| e.total_stake()).unwrap_or(0);
+            let voted_stake = {
+                let slot_tracker = vote_tracker.get_slot_vote_tracker(*optimistic_slot);
+                let r_slot_tracker = slot_tracker.as_ref().map(|s| s.read().unwrap());
+                let voted_stake = r_slot_tracker
+                    .as_ref()
+                    .and_then(|s| s.optimistic_votes_tracker(hash))
+                    .map(|s| s.stake())
+                    .unwrap_or(0);
+
+                warn!(
+                    "Optimistic slot {}, hash: {}, epoch: {} was not rooted,
+                    voted keys: {:?},
+                    root: {},
+                    root bank hash: {},
+                    voted stake: {},
+                    total epoch stake: {},
+                    pct: {}",
+                    optimistic_slot,
+                    hash,
+                    epoch,
+                    r_slot_tracker
+                        .as_ref()
+                        .and_then(|s| s.optimistic_votes_tracker(hash))
+                        .map(|s| s.voted()),
+                    root,
+                    root_bank.hash(),
+                    voted_stake,
+                    total_epoch_stake,
+                    voted_stake as f64 / total_epoch_stake as f64,
+                );
+                voted_stake
+            };
+
+            datapoint_warn!(
+                "optimistic_slot_not_rooted",
+                ("slot", *optimistic_slot, i64),
+                ("epoch", epoch, i64),
+                ("root", root, i64),
+                ("voted_stake", voted_stake, i64),
+                ("total_epoch_stake", total_epoch_stake, i64),
+            );
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::consensus::test::VoteSimulator;
+    use solana_ledger::get_tmp_ledger_path;
+    use solana_runtime::bank::Bank;
+    use solana_sdk::pubkey::Pubkey;
+    use std::collections::HashMap;
+    use trees::tr;
+
+    #[test]
+    fn test_add_new_optimistic_confirmed_slots() {
+        let snapshot_start_slot = 10;
+        let bank_hash = Hash::default();
+        let mut optimistic_confirmation_verifier =
+            OptimisticConfirmationVerifier::new(snapshot_start_slot);
+        optimistic_confirmation_verifier
+            .add_new_optimistic_confirmed_slots(vec![(snapshot_start_slot - 1, bank_hash)]);
+        optimistic_confirmation_verifier
+            .add_new_optimistic_confirmed_slots(vec![(snapshot_start_slot, bank_hash)]);
+        optimistic_confirmation_verifier
+            .add_new_optimistic_confirmed_slots(vec![(snapshot_start_slot + 1, bank_hash)]);
+        assert_eq!(optimistic_confirmation_verifier.unchecked_slots.len(), 1);
+        assert!(optimistic_confirmation_verifier
+            .unchecked_slots
+            .contains(&(snapshot_start_slot + 1, bank_hash)));
+    }
+
+    #[test]
+    fn test_get_unrooted_optimistic_slots_same_slot_different_hash() {
+        let snapshot_start_slot = 0;
+        let mut optimistic_confirmation_verifier =
+            OptimisticConfirmationVerifier::new(snapshot_start_slot);
+        let bad_bank_hash = Hash::new(&[42u8; 32]);
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            let optimistic_slots = vec![(1, bad_bank_hash), (3, Hash::default())];
+            optimistic_confirmation_verifier.add_new_optimistic_confirmed_slots(optimistic_slots);
+            let vote_simulator = setup_forks();
+            let bank1 = vote_simulator
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(1)
+                .cloned()
+                .unwrap();
+            assert_eq!(
+                optimistic_confirmation_verifier.get_unrooted_optimistic_slots(&bank1, &blockstore),
+                vec![(1, bad_bank_hash)]
+            );
+            assert_eq!(optimistic_confirmation_verifier.unchecked_slots.len(), 1);
+            assert!(optimistic_confirmation_verifier
+                .unchecked_slots
+                .contains(&(3, Hash::default())));
+        }
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    fn test_get_unrooted_optimistic_slots() {
+        let snapshot_start_slot = 0;
+        let mut optimistic_confirmation_verifier =
+            OptimisticConfirmationVerifier::new(snapshot_start_slot);
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            let mut vote_simulator = setup_forks();
+            let optimistic_slots: Vec<_> = vec![1, 3, 5]
+                .into_iter()
+                .map(|s| {
+                    (
+                        s,
+                        vote_simulator
+                            .bank_forks
+                            .read()
+                            .unwrap()
+                            .get(s)
+                            .unwrap()
+                            .hash(),
+                    )
+                })
+                .collect();
+
+            // If root is on same fork, nothing should be returned
+            optimistic_confirmation_verifier
+                .add_new_optimistic_confirmed_slots(optimistic_slots.clone());
+            let bank5 = vote_simulator
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(5)
+                .cloned()
+                .unwrap();
+            assert!(optimistic_confirmation_verifier
+                .get_unrooted_optimistic_slots(&bank5, &blockstore)
+                .is_empty());
+            // 5 is >= than all the unchecked slots, so should clear everything
+            assert!(optimistic_confirmation_verifier.unchecked_slots.is_empty());
+
+            // If root is on same fork, nothing should be returned
+            optimistic_confirmation_verifier
+                .add_new_optimistic_confirmed_slots(optimistic_slots.clone());
+            let bank3 = vote_simulator
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(3)
+                .cloned()
+                .unwrap();
+            assert!(optimistic_confirmation_verifier
+                .get_unrooted_optimistic_slots(&bank3, &blockstore)
+                .is_empty());
+            // 3 is bigger than only slot 1, so slot 5 should be left over
+            assert_eq!(optimistic_confirmation_verifier.unchecked_slots.len(), 1);
+            assert!(optimistic_confirmation_verifier
+                .unchecked_slots
+                .contains(&optimistic_slots[2]));
+
+            // If root is on different fork, the slots < root on different fork should
+            // be returned
+            optimistic_confirmation_verifier
+                .add_new_optimistic_confirmed_slots(optimistic_slots.clone());
+            let bank4 = vote_simulator
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(4)
+                .cloned()
+                .unwrap();
+            assert_eq!(
+                optimistic_confirmation_verifier.get_unrooted_optimistic_slots(&bank4, &blockstore),
+                vec![optimistic_slots[1]]
+            );
+            // 4 is bigger than only slots 1 and 3, so slot 5 should be left over
+            assert_eq!(optimistic_confirmation_verifier.unchecked_slots.len(), 1);
+            assert!(optimistic_confirmation_verifier
+                .unchecked_slots
+                .contains(&optimistic_slots[2]));
+
+            // Now set a root at slot 5, purging BankForks of slots < 5
+            vote_simulator.set_root(5);
+
+            // Add a new bank 7 that descends from 6
+            let bank6 = vote_simulator
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(6)
+                .cloned()
+                .unwrap();
+            vote_simulator
+                .bank_forks
+                .write()
+                .unwrap()
+                .insert(Bank::new_from_parent(&bank6, &Pubkey::default(), 7));
+            let bank7 = vote_simulator
+                .bank_forks
+                .read()
+                .unwrap()
+                .get(7)
+                .unwrap()
+                .clone();
+            assert!(!bank7.ancestors.contains_key(&3));
+
+            // Should return slots 1, 3 as part of the rooted fork because there's no
+            // ancestry information
+            optimistic_confirmation_verifier
+                .add_new_optimistic_confirmed_slots(optimistic_slots.clone());
+            assert_eq!(
+                optimistic_confirmation_verifier.get_unrooted_optimistic_slots(&bank7, &blockstore),
+                optimistic_slots[0..=1].to_vec()
+            );
+            assert!(optimistic_confirmation_verifier.unchecked_slots.is_empty());
+
+            // If we know set the root in blockstore, should return nothing
+            blockstore.set_roots(&[1, 3]).unwrap();
+            optimistic_confirmation_verifier.add_new_optimistic_confirmed_slots(optimistic_slots);
+            assert!(optimistic_confirmation_verifier
+                .get_unrooted_optimistic_slots(&bank7, &blockstore)
+                .is_empty());
+            assert!(optimistic_confirmation_verifier.unchecked_slots.is_empty());
+        }
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
+    fn setup_forks() -> VoteSimulator {
+        /*
+            Build fork structure:
+                 slot 0
+                   |
+                 slot 1
+                 /    \
+            slot 2    |
+               |    slot 3
+            slot 4    |
+                    slot 5
+                      |
+                    slot 6
+        */
+        let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5) / (tr(6)))));
+
+        let mut vote_simulator = VoteSimulator::new(1);
+        vote_simulator.fill_bank_forks(forks, &HashMap::new());
+        vote_simulator
+    }
+}

core/src/repair_service.rs

@@ -1197,6 +1197,7 @@ mod test {
             &keypairs.node_keypair,
             &keypairs.vote_keypair,
             &keypairs.vote_keypair,
+            None,
         );
         bank9.process_transaction(&vote_tx).unwrap();
         assert!(bank9.get_signature_status(&vote_tx.signatures[0]).is_some());

core/src/replay_stage.rs

@@ -2706,6 +2706,7 @@ pub(crate) mod tests {
             &my_keypairs.node_keypair,
             &my_keypairs.vote_keypair,
             &my_keypairs.vote_keypair,
+            None,
         );
 
         let bank_forks = RwLock::new(bank_forks);

core/src/rpc_pubsub.rs

@@ -968,6 +968,7 @@ mod tests {
                 node_keypair,
                 vote_keypair,
                 vote_keypair,
+                None,
             );
             votes_sender.send(vec![vote_tx]).unwrap();
         });
@@ -978,7 +979,7 @@ mod tests {
         ClusterInfoVoteListener::get_and_process_votes_for_tests(
             &votes_receiver,
             &vote_tracker,
-            0,
+            &bank,
             &rpc.subscriptions,
             &s,
             &replay_votes_receiver,

core/src/tpu.rs

@@ -81,6 +81,7 @@ impl Tpu {
             subscriptions.clone(),
             verified_vote_sender,
             replay_votes_receiver,
+            blockstore.clone(),
         );
 
         let banking_stage = BankingStage::new(

core/src/vote_stake_tracker.rs

@@ -0,0 +1,68 @@
+use solana_runtime::commitment::VOTE_THRESHOLD_SIZE;
+use solana_sdk::pubkey::Pubkey;
+use std::{collections::HashSet, sync::Arc};
+
+#[derive(Default)]
+pub struct VoteStakeTracker {
+    voted: HashSet<Arc<Pubkey>>,
+    stake: u64,
+}
+
+impl VoteStakeTracker {
+    // Returns true if the stake that has voted has just crosssed the supermajority
+    // of stake
+    pub fn add_vote_pubkey(
+        &mut self,
+        vote_pubkey: Arc<Pubkey>,
+        stake: u64,
+        total_epoch_stake: u64,
+    ) -> bool {
+        if !self.voted.contains(&vote_pubkey) {
+            self.voted.insert(vote_pubkey);
+            let ratio_before = self.stake as f64 / total_epoch_stake as f64;
+            self.stake += stake;
+            let ratio_now = self.stake as f64 / total_epoch_stake as f64;
+
+            ratio_before <= VOTE_THRESHOLD_SIZE && ratio_now > VOTE_THRESHOLD_SIZE
+        } else {
+            false
+        }
+    }
+
+    pub fn voted(&self) -> &HashSet<Arc<Pubkey>> {
+        &self.voted
+    }
+
+    pub fn stake(&self) -> u64 {
+        self.stake
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_add_vote_pubkey() {
+        let total_epoch_stake = 10;
+        let mut vote_stake_tracker = VoteStakeTracker::default();
+        for i in 0..10 {
+            let pubkey = Arc::new(Pubkey::new_rand());
+            let res = vote_stake_tracker.add_vote_pubkey(pubkey.clone(), 1, total_epoch_stake);
+            let stake = vote_stake_tracker.stake();
+            vote_stake_tracker.add_vote_pubkey(pubkey.clone(), 1, total_epoch_stake);
+            let stake2 = vote_stake_tracker.stake();
+
+            // Stake should not change from adding same pubkey twice
+            assert_eq!(stake, stake2);
+
+            // at i == 7, the voted stake is 70%, which is the first time crossing
+            // the supermajority threshold
+            if i == 6 {
+                assert!(res);
+            } else {
+                assert!(!res);
+            }
+        }
+    }
+}

core/tests/ledger_cleanup.rs

@@ -171,7 +171,7 @@ mod tests {
     }
 
     fn emit_stats(
-        time_initial: &Instant,
+        time_initial: Instant,
         time_previous: &mut Instant,
         storage_previous: &mut u64,
         start_slot: u64,
@@ -187,7 +187,7 @@ mod tests {
 
         println!(
             "{},{},{},{},{},{},{},{},{},{},{}",
-            time_now.duration_since(*time_initial).as_millis(),
+            time_now.duration_since(time_initial).as_millis(),
             time_now.duration_since(*time_previous).as_millis(),
             start_slot,
             batch_size,
@@ -249,7 +249,7 @@ mod tests {
 
         emit_header();
         emit_stats(
-            &time_initial,
+            time_initial,
             &mut time_previous,
             &mut storage_previous,
             0,
@@ -273,7 +273,7 @@ mod tests {
             sender.send(x).unwrap();
 
             emit_stats(
-                &time_initial,
+                time_initial,
                 &mut time_previous,
                 &mut storage_previous,
                 x,
@@ -303,7 +303,7 @@ mod tests {
         sender.send(benchmark_slots).unwrap();
 
         emit_stats(
-            &time_initial,
+            time_initial,
             &mut time_previous,
             &mut storage_previous,
             benchmark_slots,
@@ -324,7 +324,7 @@ mod tests {
         }
 
         emit_stats(
-            &time_initial,
+            time_initial,
             &mut time_previous,
             &mut storage_previous,
             benchmark_slots,

metrics/scripts/grafana-provisioning/dashboards/cluster-monitor.json

@@ -6206,7 +6206,7 @@
         "x": 16,
         "y": 48
       },
-      "id": 73,
+      "id": 40,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -7120,7 +7120,7 @@
         "x": 16,
         "y": 54
       },
-      "id": 40,
+      "id": 43,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -7707,7 +7707,7 @@
         "x": 16,
         "y": 60
       },
-      "id": 43,
+      "id": 46,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -8495,7 +8495,7 @@
         "x": 16,
         "y": 65
       },
-      "id": 46,
+      "id": 49,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -8613,7 +8613,7 @@
         "x": 16,
         "y": 71
       },
-      "id": 49,
+      "id": 50,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -8885,7 +8885,7 @@
         "x": 0,
         "y": 77
       },
-      "id": 50,
+      "id": 51,
       "panels": [],
       "title": "Tower Consensus",
       "type": "row"
@@ -8908,7 +8908,7 @@
         "x": 0,
         "y": 78
       },
-      "id": 51,
+      "id": 52,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -9068,7 +9068,7 @@
         "x": 8,
         "y": 78
       },
-      "id": 52,
+      "id": 53,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -9228,7 +9228,7 @@
         "x": 16,
         "y": 78
       },
-      "id": 53,
+      "id": 54,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -9405,15 +9405,384 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {
+        "cluster-info.repair": "#ba43a9",
+        "replay_stage-new_leader.last": "#00ffbb",
+        "tower-vote.last": "#00ffbb",
+        "window-service.receive": "#b7dbab",
+        "window-stage.consumed": "#5195ce"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 0,
+        "y": 83
+      },
+      "id": 74,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": false,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "hide": false,
+          "measurement": "cluster_info-vote-count",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "query": "SELECT last(\"slot\") FROM \"$testnet\".\"autogen\".\"optimistic_slot\" WHERE host_id::tag =~ /$hostid/ AND $timeFilter GROUP BY time($__interval)",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "count"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Optimistic Slots ($hostid)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "none",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "cluster-info.repair": "#ba43a9",
+        "replay_stage-new_leader.last": "#00ffbb",
+        "tower-vote.last": "#00ffbb",
+        "window-service.receive": "#b7dbab",
+        "window-stage.consumed": "#5195ce"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 8,
+        "y": 83
+      },
+      "id": 75,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": false,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "hide": false,
+          "measurement": "cluster_info-vote-count",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "query": "SELECT last(\"slot\") FROM \"$testnet\".\"autogen\".\"optimistic_slot_not_rooted\" WHERE host_id::tag =~ /$hostid/ AND $timeFilter GROUP BY time($__interval)",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "count"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Unrooted Optimistic Slots ($hostid)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "none",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "cluster-info.repair": "#ba43a9",
+        "replay_stage-new_leader.last": "#00ffbb",
+        "tower-vote.last": "#00ffbb",
+        "window-service.receive": "#b7dbab",
+        "window-stage.consumed": "#5195ce"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$datasource",
+      "fill": 1,
+      "gridPos": {
+        "h": 5,
+        "w": 8,
+        "x": 16,
+        "y": 83
+      },
+      "id": 76,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": false,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "hide": false,
+          "measurement": "cluster_info-vote-count",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "query": "SELECT last(\"average_elapsed_ms\") FROM \"$testnet\".\"autogen\".\"optimistic_slot_elapsed\" WHERE host_id::tag =~ /$hostid/ AND $timeFilter GROUP BY time($__interval)",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "count"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Average Time Between Optimistic Confirmations ($hostid)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "none",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 83
+        "y": 88
       },
-      "id": 54,
+      "id": 55,
       "panels": [],
       "repeat": null,
       "title": "IP Network",
@@ -9430,9 +9799,9 @@
         "h": 5,
         "w": 12,
         "x": 0,
-        "y": 84
+        "y": 89
       },
-      "id": 55,
+      "id": 56,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -9663,9 +10032,9 @@
         "h": 5,
         "w": 12,
         "x": 12,
-        "y": 84
+        "y": 89
       },
-      "id": 56,
+      "id": 57,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -9816,9 +10185,9 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 89
+        "y": 94
       },
-      "id": 57,
+      "id": 58,
       "panels": [],
       "title": "Signature Verification",
       "type": "row"
@@ -9834,9 +10203,9 @@
         "h": 5,
         "w": 12,
         "x": 0,
-        "y": 90
+        "y": 95
       },
-      "id": 58,
+      "id": 59,
       "legend": {
         "avg": false,
         "current": false,
@@ -10036,9 +10405,9 @@
         "h": 5,
         "w": 12,
         "x": 12,
-        "y": 90
+        "y": 95
       },
-      "id": 59,
+      "id": 60,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -10185,9 +10554,9 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 95
+        "y": 100
       },
-      "id": 60,
+      "id": 61,
       "panels": [],
       "title": "Snapshots",
       "type": "row"
@@ -10203,9 +10572,9 @@
         "h": 6,
         "w": 8,
         "x": 0,
-        "y": 96
+        "y": 101
       },
-      "id": 61,
+      "id": 62,
       "legend": {
         "avg": false,
         "current": false,
@@ -10395,9 +10764,9 @@
         "h": 6,
         "w": 8,
         "x": 8,
-        "y": 96
+        "y": 101
       },
-      "id": 62,
+      "id": 63,
       "legend": {
         "avg": false,
         "current": false,
@@ -10663,9 +11032,9 @@
         "h": 6,
         "w": 8,
         "x": 16,
-        "y": 96
+        "y": 101
       },
-      "id": 63,
+      "id": 64,
       "legend": {
         "avg": false,
         "current": false,
@@ -10857,9 +11226,9 @@
         "h": 6,
         "w": 8,
         "x": 0,
-        "y": 102
+        "y": 107
       },
-      "id": 64,
+      "id": 65,
       "legend": {
         "avg": false,
         "current": false,
@@ -11048,9 +11417,9 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 108
+        "y": 113
       },
-      "id": 65,
+      "id": 66,
       "panels": [],
       "title": "RPC Send Transaction Service",
       "type": "row"
@@ -11066,9 +11435,9 @@
         "h": 6,
         "w": 12,
         "x": 0,
-        "y": 109
+        "y": 114
       },
-      "id": 66,
+      "id": 67,
       "legend": {
         "avg": false,
         "current": false,
@@ -11184,9 +11553,9 @@
         "h": 6,
         "w": 12,
         "x": 12,
-        "y": 109
+        "y": 114
       },
-      "id": 67,
+      "id": 68,
       "legend": {
         "avg": false,
         "current": false,
@@ -11447,9 +11816,9 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 115
+        "y": 120
       },
-      "id": 68,
+      "id": 69,
       "panels": [],
       "title": "Bench TPS",
       "type": "row"
@@ -11465,9 +11834,9 @@
         "h": 5,
         "w": 7,
         "x": 0,
-        "y": 116
+        "y": 121
       },
-      "id": 69,
+      "id": 70,
       "legend": {
         "avg": false,
         "current": false,
@@ -11580,9 +11949,9 @@
         "h": 5,
         "w": 7,
         "x": 7,
-        "y": 116
+        "y": 121
       },
-      "id": 70,
+      "id": 71,
       "legend": {
         "alignAsTable": false,
         "avg": false,
@@ -11805,9 +12174,9 @@
         "h": 5,
         "w": 10,
         "x": 14,
-        "y": 116
+        "y": 121
       },
-      "id": 71,
+      "id": 72,
       "links": [],
       "pageSize": null,
       "scroll": true,
@@ -11893,9 +12262,9 @@
         "h": 4,
         "w": 10,
         "x": 0,
-        "y": 121
+        "y": 126
       },
-      "id": 72,
+      "id": 73,
       "legend": {
         "avg": false,
         "current": false,
@@ -11998,7 +12367,7 @@
       }
     }
   ],
-  "refresh": "1m",
+  "refresh": "60s",
   "schemaVersion": 16,
   "style": "dark",
   "tags": [],

programs/vote/src/vote_transaction.rs

@@ -14,13 +14,23 @@ pub fn new_vote_transaction(
     node_keypair: &Keypair,
     vote_keypair: &Keypair,
     authorized_voter_keypair: &Keypair,
+    switch_proof_hash: Option<Hash>,
 ) -> Transaction {
     let votes = Vote::new(slots, bank_hash);
-    let vote_ix = vote_instruction::vote(
+    let vote_ix = if let Some(switch_proof_hash) = switch_proof_hash {
+        vote_instruction::vote_switch(
             &vote_keypair.pubkey(),
             &authorized_voter_keypair.pubkey(),
             votes,
-    );
+            switch_proof_hash,
+        )
+    } else {
+        vote_instruction::vote(
+            &vote_keypair.pubkey(),
+            &authorized_voter_keypair.pubkey(),
+            votes,
+        )
+    };
 
     let mut vote_tx = Transaction::new_with_payer(&[vote_ix], Some(&node_keypair.pubkey()));