solana/local-cluster/tests/local_cluster.rs

use assert_matches::assert_matches;
use log::*;
use serial_test_derive::serial;
use solana_client::rpc_client::RpcClient;
use solana_client::thin_client::create_client;
use solana_core::{
    broadcast_stage::BroadcastStageType,
    consensus::{SWITCH_FORK_THRESHOLD, VOTE_THRESHOLD_DEPTH},
    gossip_service::discover_cluster,
    validator::ValidatorConfig,
};
use solana_download_utils::download_snapshot;
use solana_ledger::{
    blockstore::Blockstore, leader_schedule::FixedSchedule, leader_schedule::LeaderSchedule,
};
use solana_local_cluster::{
    cluster::Cluster,
    cluster_tests,
    local_cluster::{ClusterConfig, LocalCluster},
};
use solana_runtime::{
    bank_forks::{CompressionType, SnapshotConfig},
    snapshot_utils,
};
use solana_sdk::{
    client::{AsyncClient, SyncClient},
    clock::{self, Slot},
    commitment_config::CommitmentConfig,
    epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
    genesis_config::OperatingMode,
    hash::Hash,
    poh_config::PohConfig,
    pubkey::Pubkey,
    signature::{Keypair, Signer},
};
use std::sync::atomic::{AtomicBool, Ordering};
use std::{
    collections::{HashMap, HashSet},
    fs, iter,
    path::{Path, PathBuf},
    sync::Arc,
    thread::sleep,
    time::Duration,
};
use tempfile::TempDir;

#[test]
#[serial]
fn test_ledger_cleanup_service() {
    solana_logger::setup();
    error!("test_ledger_cleanup_service");
    let num_nodes = 3;
    let mut validator_config = ValidatorConfig::default();
    validator_config.max_ledger_shreds = Some(100);
    let config = ClusterConfig {
        cluster_lamports: 10_000,
        poh_config: PohConfig::new_sleep(Duration::from_millis(50)),
        node_stakes: vec![100; num_nodes],
        validator_configs: vec![validator_config; num_nodes],
        ..ClusterConfig::default()
    };
    let mut cluster = LocalCluster::new(&config);
    // 200ms/per * 100 = 20 seconds, so sleep a little longer than that.
    sleep(Duration::from_secs(60));

    cluster_tests::spend_and_verify_all_nodes(
        &cluster.entry_point_info,
        &cluster.funding_keypair,
        num_nodes,
        HashSet::new(),
    );
    cluster.close_preserve_ledgers();
    //check everyone's ledgers and make sure only ~100 slots are stored
    for info in cluster.validators.values() {
        let mut slots = 0;
        let blockstore = Blockstore::open(&info.info.ledger_path).unwrap();
        blockstore
            .slot_meta_iterator(0)
            .unwrap()
            .for_each(|_| slots += 1);
        // with 3 nodes up to 3 slots can be in progress and not complete so max slots in blockstore should be up to 103
        assert!(slots <= 103, "got {}", slots);
    }
}

#[test]
#[serial]
fn test_spend_and_verify_all_nodes_1() {
    solana_logger::setup();
    error!("test_spend_and_verify_all_nodes_1");
    let num_nodes = 1;
    let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
    cluster_tests::spend_and_verify_all_nodes(
        &local.entry_point_info,
        &local.funding_keypair,
        num_nodes,
        HashSet::new(),
    );
}

#[test]
#[serial]
fn test_spend_and_verify_all_nodes_2() {
    solana_logger::setup();
    error!("test_spend_and_verify_all_nodes_2");
    let num_nodes = 2;
    let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
    cluster_tests::spend_and_verify_all_nodes(
        &local.entry_point_info,
        &local.funding_keypair,
        num_nodes,
        HashSet::new(),
    );
}

#[test]
#[serial]
fn test_spend_and_verify_all_nodes_3() {
    solana_logger::setup();
    error!("test_spend_and_verify_all_nodes_3");
    let num_nodes = 3;
    let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
    cluster_tests::spend_and_verify_all_nodes(
        &local.entry_point_info,
        &local.funding_keypair,
        num_nodes,
        HashSet::new(),
    );
}

#[test]
#[allow(unused_attributes)]
#[ignore]
fn test_spend_and_verify_all_nodes_env_num_nodes() {
    solana_logger::setup();
    let num_nodes: usize = std::env::var("NUM_NODES")
        .expect("please set environment variable NUM_NODES")
        .parse()
        .expect("could not parse NUM_NODES as a number");
    let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
    cluster_tests::spend_and_verify_all_nodes(
        &local.entry_point_info,
        &local.funding_keypair,
        num_nodes,
        HashSet::new(),
    );
}

#[allow(unused_attributes)]
#[test]
#[should_panic]
fn test_validator_exit_default_config_should_panic() {
    solana_logger::setup();
    error!("test_validator_exit_default_config_should_panic");
    let num_nodes = 2;
    let local = LocalCluster::new_with_equal_stakes(num_nodes, 10_000, 100);
    cluster_tests::validator_exit(&local.entry_point_info, num_nodes);
}

#[test]
#[serial]
fn test_validator_exit_2() {
    solana_logger::setup();
    error!("test_validator_exit_2");
    let num_nodes = 2;
    let mut validator_config = ValidatorConfig::default();
    validator_config.rpc_config.enable_validator_exit = true;
    validator_config.wait_for_supermajority = Some(0);

    let config = ClusterConfig {
        cluster_lamports: 10_000,
        node_stakes: vec![100; num_nodes],
        validator_configs: vec![validator_config; num_nodes],
        ..ClusterConfig::default()
    };
    let local = LocalCluster::new(&config);
    cluster_tests::validator_exit(&local.entry_point_info, num_nodes);
}

// Cluster needs a supermajority to remain, so the minimum size for this test is 4
#[test]
#[serial]
fn test_leader_failure_4() {
    solana_logger::setup();
    error!("test_leader_failure_4");
    let num_nodes = 4;
    let mut validator_config = ValidatorConfig::default();
    validator_config.rpc_config.enable_validator_exit = true;
    let config = ClusterConfig {
        cluster_lamports: 10_000,
        node_stakes: vec![100; 4],
        validator_configs: vec![validator_config; num_nodes],
        ..ClusterConfig::default()
    };
    let local = LocalCluster::new(&config);
    cluster_tests::kill_entry_and_spend_and_verify_rest(
        &local.entry_point_info,
        &local.funding_keypair,
        num_nodes,
        config.ticks_per_slot * config.poh_config.target_tick_duration.as_millis() as u64,
    );
}

/// This function runs a network, initiates a partition based on a
/// configuration, resolve the partition, then checks that the network
/// continues to achieve consensus
/// # Arguments
/// * `partitions` - A slice of partition configurations, where each partition
/// configuration is a slice of (usize, bool), representing a node's stake and
/// whether or not it should be killed during the partition
/// * `leader_schedule` - An option that specifies whether the cluster should
/// run with a fixed, predetermined leader schedule
#[allow(clippy::cognitive_complexity)]
fn run_cluster_partition<E, F>(
    partitions: &[&[usize]],
    leader_schedule: Option<(LeaderSchedule, Vec<Arc<Keypair>>)>,
    on_partition_start: E,
    on_partition_resolved: F,
) where
    E: Fn(&mut LocalCluster),
    F: Fn(&mut LocalCluster),
{
    solana_logger::setup();
    info!("PARTITION_TEST!");
    let num_nodes = partitions.len();
    let node_stakes: Vec<_> = partitions
        .iter()
        .flat_map(|p| p.iter().map(|stake_weight| 100 * *stake_weight as u64))
        .collect();
    assert_eq!(node_stakes.len(), num_nodes);
    let cluster_lamports = node_stakes.iter().sum::<u64>() * 2;
    let enable_partition = Arc::new(AtomicBool::new(true));
    let mut validator_config = ValidatorConfig::default();
    validator_config.enable_partition = Some(enable_partition.clone());

    // Returns:
    // 1) The keys for the validators
    // 2) The amount of time it would take to iterate through one full iteration of the given
    // leader schedule
    let (validator_keys, leader_schedule_time): (Vec<_>, u64) = {
        if let Some((leader_schedule, validator_keys)) = leader_schedule {
            assert_eq!(validator_keys.len(), num_nodes);
            let num_slots_per_rotation = leader_schedule.num_slots() as u64;
            let fixed_schedule = FixedSchedule {
                start_epoch: 0,
                leader_schedule: Arc::new(leader_schedule),
            };
            validator_config.fixed_leader_schedule = Some(fixed_schedule);
            (
                validator_keys,
                num_slots_per_rotation * clock::DEFAULT_MS_PER_SLOT,
            )
        } else {
            (
                iter::repeat_with(|| Arc::new(Keypair::new()))
                    .take(partitions.len())
                    .collect(),
                10_000,
            )
        }
    };

    let slots_per_epoch = 2048;
    let config = ClusterConfig {
        cluster_lamports,
        node_stakes,
        validator_configs: vec![validator_config; num_nodes],
        validator_keys: Some(
            validator_keys
                .into_iter()
                .zip(iter::repeat_with(|| true))
                .collect(),
        ),
        slots_per_epoch,
        stakers_slot_offset: slots_per_epoch,
        skip_warmup_slots: true,
        ..ClusterConfig::default()
    };

    info!(
        "PARTITION_TEST starting cluster with {:?} partitions slots_per_epoch: {}",
        partitions, config.slots_per_epoch,
    );
    let mut cluster = LocalCluster::new(&config);

    info!("PARTITION_TEST spend_and_verify_all_nodes(), ensure all nodes are caught up");
    cluster_tests::spend_and_verify_all_nodes(
        &cluster.entry_point_info,
        &cluster.funding_keypair,
        num_nodes,
        HashSet::new(),
    );

    let cluster_nodes = discover_cluster(&cluster.entry_point_info.gossip, num_nodes).unwrap();

    // Check epochs have correct number of slots
    info!("PARTITION_TEST sleeping until partition starting condition",);
    for node in &cluster_nodes {
        let node_client = RpcClient::new_socket(node.rpc);
        let epoch_info = node_client.get_epoch_info().unwrap();
        info!("slots_per_epoch: {:?}", epoch_info);
        assert_eq!(epoch_info.slots_in_epoch, slots_per_epoch);
    }

    info!("PARTITION_TEST start partition");
    enable_partition.store(false, Ordering::Relaxed);
    on_partition_start(&mut cluster);

    sleep(Duration::from_millis(leader_schedule_time));

    info!("PARTITION_TEST remove partition");
    enable_partition.store(true, Ordering::Relaxed);

    // Give partitions time to propagate their blocks from during the partition
    // after the partition resolves
    let timeout = 10_000;
    let propagation_time = leader_schedule_time;
    info!(
        "PARTITION_TEST resolving partition. sleeping {} ms",
        timeout
    );
    sleep(Duration::from_millis(timeout));
    info!(
        "PARTITION_TEST waiting for blocks to propagate after partition {}ms",
        propagation_time
    );
    sleep(Duration::from_millis(propagation_time));
    info!("PARTITION_TEST resuming normal operation");
    on_partition_resolved(&mut cluster);
}

#[allow(unused_attributes)]
#[ignore]
#[test]
#[serial]
fn test_cluster_partition_1_2() {
    let empty = |_: &mut LocalCluster| {};
    let on_partition_resolved = |cluster: &mut LocalCluster| {
        cluster.check_for_new_roots(16, &"PARTITION_TEST");
    };
    run_cluster_partition(&[&[1], &[1, 1]], None, empty, on_partition_resolved)
}

#[allow(unused_attributes)]
#[ignore]
#[test]
#[serial]
fn test_cluster_partition_1_1() {
    let empty = |_: &mut LocalCluster| {};
    let on_partition_resolved = |cluster: &mut LocalCluster| {
        cluster.check_for_new_roots(16, &"PARTITION_TEST");
    };
    run_cluster_partition(&[&[1], &[1]], None, empty, on_partition_resolved)
}

#[test]
#[serial]
fn test_cluster_partition_1_1_1() {
    let empty = |_: &mut LocalCluster| {};
    let on_partition_resolved = |cluster: &mut LocalCluster| {
        cluster.check_for_new_roots(16, &"PARTITION_TEST");
    };
    run_cluster_partition(&[&[1], &[1], &[1]], None, empty, on_partition_resolved)
}

#[test]
#[serial]
fn test_kill_heaviest_partition() {
    // This test:
    // 1) Spins up four partitions, the heaviest being the first with more stake
    // 2) Schedules the other validators for sufficient slots in the schedule
    // so that they will still be locked out of voting for the major partition
    // when the partition resolves
    // 3) Kills the most staked partition. Validators are locked out, but should all
    // eventually choose the major partition
    // 4) Check for recovery
    let mut leader_schedule = vec![];
    let num_slots_per_validator = 8;
    let partitions: [&[usize]; 4] = [&[11], &[10], &[10], &[10]];
    let validator_keys: Vec<_> = iter::repeat_with(|| Arc::new(Keypair::new()))
        .take(partitions.len())
        .collect();
    for (i, k) in validator_keys.iter().enumerate() {
        let num_slots = {
            if i == 0 {
                // Set up the leader to have 50% of the slots
                num_slots_per_validator * (partitions.len() - 1)
            } else {
                num_slots_per_validator
            }
        };
        for _ in 0..num_slots {
            leader_schedule.push(k.pubkey())
        }
    }
    info!("leader_schedule: {}", leader_schedule.len());

    let empty = |_: &mut LocalCluster| {};
    let validator_to_kill = validator_keys[0].pubkey();
    let on_partition_resolved = |cluster: &mut LocalCluster| {
        info!("Killing validator with id: {}", validator_to_kill);
        cluster.exit_node(&validator_to_kill);
        cluster.check_for_new_roots(16, &"PARTITION_TEST");
    };
    run_cluster_partition(
        &partitions,
        Some((
            LeaderSchedule::new_from_schedule(leader_schedule),
            validator_keys,
        )),
        empty,
        on_partition_resolved,
    )
}

#[allow(clippy::assertions_on_constants)]
fn run_kill_partition_switch_threshold<F>(
    failures_stake: u64,
    alive_stake_1: u64,
    alive_stake_2: u64,
    on_partition_resolved: F,
) where
    F: Fn(&mut LocalCluster),
{
    // Needs to be at least 1/3 or there will be no overlap
    // with the confirmation supermajority 2/3
    assert!(SWITCH_FORK_THRESHOLD >= 1f64 / 3f64);
    info!(
        "stakes: {} {} {}",
        failures_stake, alive_stake_1, alive_stake_2
    );

    // This test:
    // 1) Spins up three partitions
    // 2) Kills the first partition with the stake `failures_stake`
    // 5) runs `on_partition_resolved`
    let mut leader_schedule = vec![];
    let num_slots_per_validator = 8;
    let partitions: [&[usize]; 3] = [
        &[(failures_stake as usize)],
        &[(alive_stake_1 as usize)],
        &[(alive_stake_2 as usize)],
    ];
    let validator_keys: Vec<_> = iter::repeat_with(|| Arc::new(Keypair::new()))
        .take(partitions.len())
        .collect();
    for (i, k) in validator_keys.iter().enumerate() {
        let num_slots = {
            if i == 0 {
                // Set up the leader to have 50% of the slots
                num_slots_per_validator * (partitions.len() - 1)
            } else {
                num_slots_per_validator
            }
        };
        for _ in 0..num_slots {
            leader_schedule.push(k.pubkey())
        }
    }
    info!("leader_schedule: {}", leader_schedule.len());

    let validator_to_kill = validator_keys[0].pubkey();
    let on_partition_start = |cluster: &mut LocalCluster| {
        info!("Killing validator with id: {}", validator_to_kill);
        cluster.exit_node(&validator_to_kill);
    };
    run_cluster_partition(
        &partitions,
        Some((
            LeaderSchedule::new_from_schedule(leader_schedule),
            validator_keys,
        )),
        on_partition_start,
        on_partition_resolved,
    )
}

#[test]
#[serial]
fn test_kill_partition_switch_threshold_no_progress() {
    let max_switch_threshold_failure_pct = 1.0 - 2.0 * SWITCH_FORK_THRESHOLD;
    let total_stake = 10_000;
    let max_failures_stake = (max_switch_threshold_failure_pct * total_stake as f64) as u64;

    let failures_stake = max_failures_stake;
    let total_alive_stake = total_stake - failures_stake;
    let alive_stake_1 = total_alive_stake / 2;
    let alive_stake_2 = total_alive_stake - alive_stake_1;

    // Check that no new roots were set 400 slots after partition resolves (gives time
    // for lockouts built during partition to resolve and gives validators an opportunity
    // to try and switch forks)
    let on_partition_resolved = |cluster: &mut LocalCluster| {
        cluster.check_no_new_roots(400, &"PARTITION_TEST");
    };

    // This kills `max_failures_stake`, so no progress should be made
    run_kill_partition_switch_threshold(
        failures_stake,
        alive_stake_1,
        alive_stake_2,
        on_partition_resolved,
    );
}

#[test]
#[serial]
fn test_kill_partition_switch_threshold_progress() {
    let max_switch_threshold_failure_pct = 1.0 - 2.0 * SWITCH_FORK_THRESHOLD;
    let total_stake = 10_000;

    // Kill `< max_failures_stake` of the validators
    let max_failures_stake = (max_switch_threshold_failure_pct * total_stake as f64) as u64;
    let failures_stake = max_failures_stake - 1;
    let total_alive_stake = total_stake - failures_stake;

    // Partition the remaining alive validators, should still make progress
    // once the partition resolves
    let alive_stake_1 = total_alive_stake / 2;
    let alive_stake_2 = total_alive_stake - alive_stake_1;
    let bigger = std::cmp::max(alive_stake_1, alive_stake_2);
    let smaller = std::cmp::min(alive_stake_1, alive_stake_2);

    // At least one of the forks must have > SWITCH_FORK_THRESHOLD in order
    // to guarantee switching proofs can be created. Make sure the other fork
    // is <= SWITCH_FORK_THRESHOLD to make sure progress can be made. Caches
    // bugs such as liveness issues bank-weighted fork choice, which may stall
    // because the fork with less stake could have more weight, but other fork would:
    // 1) Not be able to generate a switching proof
    // 2) Other more staked fork stops voting, so doesn't catch up in bank weight.
    assert!(
        bigger as f64 / total_stake as f64 > SWITCH_FORK_THRESHOLD
            && smaller as f64 / total_stake as f64 <= SWITCH_FORK_THRESHOLD
    );

    let on_partition_resolved = |cluster: &mut LocalCluster| {
        cluster.check_for_new_roots(16, &"PARTITION_TEST");
    };
    run_kill_partition_switch_threshold(
        failures_stake,
        alive_stake_1,
        alive_stake_2,
        on_partition_resolved,
    );
}

#[test]
#[serial]
fn test_two_unbalanced_stakes() {
    solana_logger::setup();
    error!("test_two_unbalanced_stakes");
    let mut validator_config = ValidatorConfig::default();
    let num_ticks_per_second = 100;
    let num_ticks_per_slot = 10;
    let num_slots_per_epoch = MINIMUM_SLOTS_PER_EPOCH as u64;

    validator_config.rpc_config.enable_validator_exit = true;
    let mut cluster = LocalCluster::new(&ClusterConfig {
        node_stakes: vec![999_990, 3],
        cluster_lamports: 1_000_000,
        validator_configs: vec![validator_config; 2],
        ticks_per_slot: num_ticks_per_slot,
        slots_per_epoch: num_slots_per_epoch,
        stakers_slot_offset: num_slots_per_epoch,
        poh_config: PohConfig::new_sleep(Duration::from_millis(1000 / num_ticks_per_second)),
        ..ClusterConfig::default()
    });

    cluster_tests::sleep_n_epochs(
        10.0,
        &cluster.genesis_config.poh_config,
        num_ticks_per_slot,
        num_slots_per_epoch,
    );
    cluster.close_preserve_ledgers();
    let leader_pubkey = cluster.entry_point_info.id;
    let leader_ledger = cluster.validators[&leader_pubkey].info.ledger_path.clone();
    cluster_tests::verify_ledger_ticks(&leader_ledger, num_ticks_per_slot as usize);
}

#[test]
#[serial]
fn test_forwarding() {
    // Set up a cluster where one node is never the leader, so all txs sent to this node
    // will be have to be forwarded in order to be confirmed
    let config = ClusterConfig {
        node_stakes: vec![999_990, 3],
        cluster_lamports: 2_000_000,
        validator_configs: vec![ValidatorConfig::default(); 2],
        ..ClusterConfig::default()
    };
    let cluster = LocalCluster::new(&config);

    let cluster_nodes = discover_cluster(&cluster.entry_point_info.gossip, 2).unwrap();
    assert!(cluster_nodes.len() >= 2);

    let leader_pubkey = cluster.entry_point_info.id;

    let validator_info = cluster_nodes
        .iter()
        .find(|c| c.id != leader_pubkey)
        .unwrap();

    // Confirm that transactions were forwarded to and processed by the leader.
    cluster_tests::send_many_transactions(&validator_info, &cluster.funding_keypair, 10, 20);
}

#[test]
#[serial]
fn test_restart_node() {
    solana_logger::setup();
    error!("test_restart_node");
    let slots_per_epoch = MINIMUM_SLOTS_PER_EPOCH * 2 as u64;
    let ticks_per_slot = 16;
    let validator_config = ValidatorConfig::default();
    let mut cluster = LocalCluster::new(&ClusterConfig {
        node_stakes: vec![100; 1],
        cluster_lamports: 100,
        validator_configs: vec![validator_config.clone()],
        ticks_per_slot,
        slots_per_epoch,
        stakers_slot_offset: slots_per_epoch,
        ..ClusterConfig::default()
    });
    let nodes = cluster.get_node_pubkeys();
    cluster_tests::sleep_n_epochs(
        1.0,
        &cluster.genesis_config.poh_config,
        clock::DEFAULT_TICKS_PER_SLOT,
        slots_per_epoch,
    );
    cluster.exit_restart_node(&nodes[0], validator_config);
    cluster_tests::sleep_n_epochs(
        0.5,
        &cluster.genesis_config.poh_config,
        clock::DEFAULT_TICKS_PER_SLOT,
        slots_per_epoch,
    );
    cluster_tests::send_many_transactions(
        &cluster.entry_point_info,
        &cluster.funding_keypair,
        10,
        1,
    );
}

#[test]
#[serial]
fn test_listener_startup() {
    let config = ClusterConfig {
        node_stakes: vec![100; 1],
        cluster_lamports: 1_000,
        num_listeners: 3,
        validator_configs: vec![ValidatorConfig::default(); 1],
        ..ClusterConfig::default()
    };
    let cluster = LocalCluster::new(&config);
    let cluster_nodes = discover_cluster(&cluster.entry_point_info.gossip, 4).unwrap();
    assert_eq!(cluster_nodes.len(), 4);
}

#[test]
#[serial]
fn test_stable_operating_mode() {
    solana_logger::setup();

    let config = ClusterConfig {
        operating_mode: OperatingMode::Stable,
        node_stakes: vec![100; 1],
        cluster_lamports: 1_000,
        validator_configs: vec![ValidatorConfig::default(); 1],
        ..ClusterConfig::default()
    };
    let cluster = LocalCluster::new(&config);
    let cluster_nodes = discover_cluster(&cluster.entry_point_info.gossip, 1).unwrap();
    assert_eq!(cluster_nodes.len(), 1);

    let client = create_client(
        cluster.entry_point_info.client_facing_addr(),
        solana_core::cluster_info::VALIDATOR_PORT_RANGE,
    );

    // Programs that are available at epoch 0
    for program_id in [
        &solana_config_program::id(),
        &solana_sdk::system_program::id(),
        &solana_stake_program::id(),
        &solana_vote_program::id(),
    ]
    .iter()
    {
        assert_matches!(
            (
                program_id,
                client
                    .get_account_with_commitment(program_id, CommitmentConfig::recent())
                    .unwrap()
            ),
            (_program_id, Some(_))
        );
    }

    // Programs that are not available at epoch 0
    for program_id in [&solana_sdk::bpf_loader::id(), &solana_vest_program::id()].iter() {
        assert_eq!(
            (
                program_id,
                client
                    .get_account_with_commitment(program_id, CommitmentConfig::recent())
                    .unwrap()
            ),
            (program_id, None)
        );
    }
}

fn generate_frozen_account_panic(mut cluster: LocalCluster, frozen_account: Arc<Keypair>) {
    let client = cluster
        .get_validator_client(&frozen_account.pubkey())
        .unwrap();

    // Check the validator is alive by poking it over RPC
    trace!(
        "validator slot: {}",
        client
            .get_slot_with_commitment(CommitmentConfig::recent())
            .expect("get slot")
    );

    // Reset the frozen account panic signal
    solana_runtime::accounts_db::FROZEN_ACCOUNT_PANIC.store(false, Ordering::Relaxed);

    // Wait for the frozen account panic signal
    let mut i = 0;
    while !solana_runtime::accounts_db::FROZEN_ACCOUNT_PANIC.load(Ordering::Relaxed) {
        // Transfer from frozen account
        let (blockhash, _fee_calculator, _last_valid_slot) = client
            .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
            .unwrap();
        client
            .async_transfer(1, &frozen_account, &Pubkey::new_rand(), blockhash)
            .unwrap();

        sleep(Duration::from_secs(1));
        i += 1;
        if i > 10 {
            panic!("FROZEN_ACCOUNT_PANIC still false");
        }
    }

    // The validator is now broken and won't shutdown properly.  Avoid LocalCluster panic in Drop
    // with some manual cleanup:
    cluster.exit();
    cluster.validators = HashMap::default();
}

#[test]
#[serial]
fn test_frozen_account_from_genesis() {
    solana_logger::setup();
    let validator_identity =
        Arc::new(solana_sdk::signature::keypair_from_seed(&[0u8; 32]).unwrap());

    let config = ClusterConfig {
        validator_keys: Some(vec![(validator_identity.clone(), true)]),
        node_stakes: vec![100; 1],
        cluster_lamports: 1_000,
        validator_configs: vec![
            ValidatorConfig {
                // Freeze the validator identity account
                frozen_accounts: vec![validator_identity.pubkey()],
                ..ValidatorConfig::default()
            };
            1
        ],
        ..ClusterConfig::default()
    };
    generate_frozen_account_panic(LocalCluster::new(&config), validator_identity);
}

#[test]
#[serial]
fn test_frozen_account_from_snapshot() {
    solana_logger::setup();
    let validator_identity =
        Arc::new(solana_sdk::signature::keypair_from_seed(&[0u8; 32]).unwrap());

    let mut snapshot_test_config = setup_snapshot_validator_config(5, 1);
    // Freeze the validator identity account
    snapshot_test_config.validator_config.frozen_accounts = vec![validator_identity.pubkey()];

    let config = ClusterConfig {
        validator_keys: Some(vec![(validator_identity.clone(), true)]),
        node_stakes: vec![100; 1],
        cluster_lamports: 1_000,
        validator_configs: vec![snapshot_test_config.validator_config.clone()],
        ..ClusterConfig::default()
    };
    let mut cluster = LocalCluster::new(&config);

    let snapshot_package_output_path = &snapshot_test_config
        .validator_config
        .snapshot_config
        .as_ref()
        .unwrap()
        .snapshot_package_output_path;

    trace!("Waiting for snapshot at {:?}", snapshot_package_output_path);
    let (archive_filename, _archive_snapshot_hash) =
        wait_for_next_snapshot(&cluster, &snapshot_package_output_path);

    trace!("Found snapshot: {:?}", archive_filename);

    // Restart the validator from a snapshot
    let validator_info = cluster.exit_node(&validator_identity.pubkey());
    cluster.restart_node(&validator_identity.pubkey(), validator_info);

    generate_frozen_account_panic(cluster, validator_identity);
}

#[test]
#[serial]
fn test_consistency_halt() {
    solana_logger::setup();
    let snapshot_interval_slots = 20;
    let num_account_paths = 1;

    // Create cluster with a leader producing bad snapshot hashes.
    let mut leader_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
    leader_snapshot_test_config
        .validator_config
        .accounts_hash_fault_injection_slots = 40;

    let validator_stake = 10_000;
    let config = ClusterConfig {
        node_stakes: vec![validator_stake],
        cluster_lamports: 100_000,
        validator_configs: vec![leader_snapshot_test_config.validator_config],
        ..ClusterConfig::default()
    };

    let mut cluster = LocalCluster::new(&config);

    sleep(Duration::from_millis(5000));
    let cluster_nodes = discover_cluster(&cluster.entry_point_info.gossip, 1).unwrap();
    info!("num_nodes: {}", cluster_nodes.len());

    // Add a validator with the leader as trusted, it should halt when it detects
    // mismatch.
    let mut validator_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);

    let mut trusted_validators = HashSet::new();
    trusted_validators.insert(cluster_nodes[0].id);

    validator_snapshot_test_config
        .validator_config
        .trusted_validators = Some(trusted_validators);
    validator_snapshot_test_config
        .validator_config
        .halt_on_trusted_validators_accounts_hash_mismatch = true;

    warn!("adding a validator");
    cluster.add_validator(
        &validator_snapshot_test_config.validator_config,
        validator_stake as u64,
        Arc::new(Keypair::new()),
        None,
    );
    let num_nodes = 2;
    assert_eq!(
        discover_cluster(&cluster.entry_point_info.gossip, num_nodes)
            .unwrap()
            .len(),
        num_nodes
    );

    // Check for only 1 node on the network.
    let mut encountered_error = false;
    loop {
        let discover = discover_cluster(&cluster.entry_point_info.gossip, 2);
        match discover {
            Err(_) => {
                encountered_error = true;
                break;
            }
            Ok(nodes) => {
                if nodes.len() < 2 {
                    encountered_error = true;
                    break;
                }
                info!("checking cluster for fewer nodes.. {:?}", nodes.len());
            }
        }
        let client = cluster
            .get_validator_client(&cluster.entry_point_info.id)
            .unwrap();
        if let Ok(slot) = client.get_slot() {
            if slot > 210 {
                break;
            }
            info!("slot: {}", slot);
        }
        sleep(Duration::from_millis(1000));
    }
    assert!(encountered_error);
}

#[test]
#[serial]
fn test_snapshot_download() {
    solana_logger::setup();
    // First set up the cluster with 1 node
    let snapshot_interval_slots = 50;
    let num_account_paths = 3;

    let leader_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
    let validator_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);

    let stake = 10_000;
    let config = ClusterConfig {
        node_stakes: vec![stake],
        cluster_lamports: 1_000_000,
        validator_configs: vec![leader_snapshot_test_config.validator_config.clone()],
        ..ClusterConfig::default()
    };

    let mut cluster = LocalCluster::new(&config);

    // Get slot after which this was generated
    let snapshot_package_output_path = &leader_snapshot_test_config
        .validator_config
        .snapshot_config
        .as_ref()
        .unwrap()
        .snapshot_package_output_path;

    trace!("Waiting for snapshot");
    let (archive_filename, archive_snapshot_hash) =
        wait_for_next_snapshot(&cluster, &snapshot_package_output_path);

    trace!("found: {:?}", archive_filename);
    let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
        &validator_snapshot_test_config.snapshot_output_path,
        &archive_snapshot_hash,
        &CompressionType::Bzip2,
    );

    // Download the snapshot, then boot a validator from it.
    download_snapshot(
        &cluster.entry_point_info.rpc,
        &validator_archive_path,
        archive_snapshot_hash,
    )
    .unwrap();

    cluster.add_validator(
        &validator_snapshot_test_config.validator_config,
        stake,
        Arc::new(Keypair::new()),
        None,
    );
}

#[allow(unused_attributes)]
#[test]
#[serial]
fn test_snapshot_restart_tower() {
    solana_logger::setup();
    // First set up the cluster with 2 nodes
    let snapshot_interval_slots = 10;
    let num_account_paths = 2;

    let leader_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
    let validator_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);

    let config = ClusterConfig {
        node_stakes: vec![10000, 10],
        cluster_lamports: 100_000,
        validator_configs: vec![
            leader_snapshot_test_config.validator_config.clone(),
            validator_snapshot_test_config.validator_config.clone(),
        ],
        ..ClusterConfig::default()
    };

    let mut cluster = LocalCluster::new(&config);

    // Let the nodes run for a while, then stop one of the validators
    sleep(Duration::from_millis(5000));
    let all_pubkeys = cluster.get_node_pubkeys();
    let validator_id = all_pubkeys
        .into_iter()
        .find(|x| *x != cluster.entry_point_info.id)
        .unwrap();
    let validator_info = cluster.exit_node(&validator_id);

    // Get slot after which this was generated
    let snapshot_package_output_path = &leader_snapshot_test_config
        .validator_config
        .snapshot_config
        .as_ref()
        .unwrap()
        .snapshot_package_output_path;

    let (archive_filename, archive_snapshot_hash) =
        wait_for_next_snapshot(&cluster, &snapshot_package_output_path);

    // Copy archive to validator's snapshot output directory
    let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
        &validator_snapshot_test_config.snapshot_output_path,
        &archive_snapshot_hash,
        &CompressionType::Bzip2,
    );
    fs::hard_link(archive_filename, &validator_archive_path).unwrap();

    // Restart validator from snapshot, the validator's tower state in this snapshot
    // will contain slots < the root bank of the snapshot. Validator should not panic.
    cluster.restart_node(&validator_id, validator_info);

    // Test cluster can still make progress and get confirmations in tower
    // Use the restarted node as the discovery point so that we get updated
    // validator's ContactInfo
    let restarted_node_info = cluster.get_contact_info(&validator_id).unwrap();
    cluster_tests::spend_and_verify_all_nodes(
        &restarted_node_info,
        &cluster.funding_keypair,
        1,
        HashSet::new(),
    );
}

#[test]
#[serial]
fn test_snapshots_blockstore_floor() {
    solana_logger::setup();
    // First set up the cluster with 1 snapshotting leader
    let snapshot_interval_slots = 10;
    let num_account_paths = 4;

    let leader_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
    let mut validator_snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);

    let snapshot_package_output_path = &leader_snapshot_test_config
        .validator_config
        .snapshot_config
        .as_ref()
        .unwrap()
        .snapshot_package_output_path;

    let config = ClusterConfig {
        node_stakes: vec![10000],
        cluster_lamports: 100_000,
        validator_configs: vec![leader_snapshot_test_config.validator_config.clone()],
        ..ClusterConfig::default()
    };

    let mut cluster = LocalCluster::new(&config);

    trace!("Waiting for snapshot tar to be generated with slot",);

    let (archive_filename, (archive_slot, archive_hash, _)) = loop {
        let archive =
            snapshot_utils::get_highest_snapshot_archive_path(&snapshot_package_output_path);
        if archive.is_some() {
            trace!("snapshot exists");
            break archive.unwrap();
        }
        sleep(Duration::from_millis(5000));
    };

    // Copy archive to validator's snapshot output directory
    let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
        &validator_snapshot_test_config.snapshot_output_path,
        &(archive_slot, archive_hash),
        &CompressionType::Bzip2,
    );
    fs::hard_link(archive_filename, &validator_archive_path).unwrap();
    let slot_floor = archive_slot;

    // Start up a new node from a snapshot
    let validator_stake = 5;

    let cluster_nodes = discover_cluster(&cluster.entry_point_info.gossip, 1).unwrap();
    let mut trusted_validators = HashSet::new();
    trusted_validators.insert(cluster_nodes[0].id);
    validator_snapshot_test_config
        .validator_config
        .trusted_validators = Some(trusted_validators);

    cluster.add_validator(
        &validator_snapshot_test_config.validator_config,
        validator_stake,
        Arc::new(Keypair::new()),
        None,
    );
    let all_pubkeys = cluster.get_node_pubkeys();
    let validator_id = all_pubkeys
        .into_iter()
        .find(|x| *x != cluster.entry_point_info.id)
        .unwrap();
    let validator_client = cluster.get_validator_client(&validator_id).unwrap();
    let mut current_slot = 0;

    // Let this validator run a while with repair
    let target_slot = slot_floor + 40;
    while current_slot <= target_slot {
        trace!("current_slot: {}", current_slot);
        if let Ok(slot) = validator_client.get_slot_with_commitment(CommitmentConfig::recent()) {
            current_slot = slot;
        } else {
            continue;
        }
        sleep(Duration::from_secs(1));
    }

    // Check the validator ledger doesn't contain any slots < slot_floor
    cluster.close_preserve_ledgers();
    let validator_ledger_path = &cluster.validators[&validator_id];
    let blockstore = Blockstore::open(&validator_ledger_path.info.ledger_path).unwrap();

    // Skip the zeroth slot in blockstore that the ledger is initialized with
    let (first_slot, _) = blockstore.slot_meta_iterator(1).unwrap().next().unwrap();

    assert_eq!(first_slot, slot_floor);
}

#[test]
#[serial]
fn test_snapshots_restart_validity() {
    solana_logger::setup();
    let snapshot_interval_slots = 10;
    let num_account_paths = 1;
    let mut snapshot_test_config =
        setup_snapshot_validator_config(snapshot_interval_slots, num_account_paths);
    let snapshot_package_output_path = &snapshot_test_config
        .validator_config
        .snapshot_config
        .as_ref()
        .unwrap()
        .snapshot_package_output_path;

    // Set up the cluster with 1 snapshotting validator
    let mut all_account_storage_dirs = vec![vec![]];

    std::mem::swap(
        &mut all_account_storage_dirs[0],
        &mut snapshot_test_config.account_storage_dirs,
    );

    let config = ClusterConfig {
        node_stakes: vec![10000],
        cluster_lamports: 100_000,
        validator_configs: vec![snapshot_test_config.validator_config.clone()],
        ..ClusterConfig::default()
    };

    // Create and reboot the node from snapshot `num_runs` times
    let num_runs = 3;
    let mut expected_balances = HashMap::new();
    let mut cluster = LocalCluster::new(&config);
    for i in 1..num_runs {
        info!("run {}", i);
        // Push transactions to one of the nodes and confirm that transactions were
        // forwarded to and processed.
        trace!("Sending transactions");
        let new_balances = cluster_tests::send_many_transactions(
            &cluster.entry_point_info,
            &cluster.funding_keypair,
            10,
            10,
        );

        expected_balances.extend(new_balances);

        wait_for_next_snapshot(&cluster, &snapshot_package_output_path);

        // Create new account paths since validator exit is not guaranteed to cleanup RPC threads,
        // which may delete the old accounts on exit at any point
        let (new_account_storage_dirs, new_account_storage_paths) =
            generate_account_paths(num_account_paths);
        all_account_storage_dirs.push(new_account_storage_dirs);
        snapshot_test_config.validator_config.account_paths = new_account_storage_paths;

        // Restart node
        trace!("Restarting cluster from snapshot");
        let nodes = cluster.get_node_pubkeys();
        cluster.exit_restart_node(&nodes[0], snapshot_test_config.validator_config.clone());

        // Verify account balances on validator
        trace!("Verifying balances");
        cluster_tests::verify_balances(expected_balances.clone(), &cluster.entry_point_info);

        // Check that we can still push transactions
        trace!("Spending and verifying");
        cluster_tests::spend_and_verify_all_nodes(
            &cluster.entry_point_info,
            &cluster.funding_keypair,
            1,
            HashSet::new(),
        );
    }
}

#[test]
#[serial]
#[allow(unused_attributes)]
#[ignore]
fn test_fail_entry_verification_leader() {
    test_faulty_node(BroadcastStageType::FailEntryVerification);
}

#[test]
#[allow(unused_attributes)]
#[ignore]
fn test_fake_shreds_broadcast_leader() {
    test_faulty_node(BroadcastStageType::BroadcastFakeShreds);
}

fn test_faulty_node(faulty_node_type: BroadcastStageType) {
    solana_logger::setup();
    let num_nodes = 2;
    let validator_config = ValidatorConfig::default();
    let mut error_validator_config = ValidatorConfig::default();
    error_validator_config.broadcast_stage_type = faulty_node_type;
    let mut validator_configs = vec![validator_config; num_nodes - 1];
    // Push a faulty_bootstrap = vec![error_validator_config];
    validator_configs.insert(0, error_validator_config);
    let node_stakes = vec![300, 100];
    assert_eq!(node_stakes.len(), num_nodes);
    let cluster_config = ClusterConfig {
        cluster_lamports: 10_000,
        node_stakes,
        validator_configs,
        slots_per_epoch: MINIMUM_SLOTS_PER_EPOCH * 2 as u64,
        stakers_slot_offset: MINIMUM_SLOTS_PER_EPOCH * 2 as u64,
        ..ClusterConfig::default()
    };

    let cluster = LocalCluster::new(&cluster_config);

    // Check for new roots
    cluster.check_for_new_roots(16, &"test_faulty_node");
}

#[test]
// Test that when a leader is leader for banks B_i..B_{i+n}, and B_i is not
// votable, then B_{i+1} still chains to B_i
fn test_no_voting() {
    solana_logger::setup();
    let mut validator_config = ValidatorConfig::default();
    validator_config.rpc_config.enable_validator_exit = true;
    validator_config.voting_disabled = true;
    let config = ClusterConfig {
        cluster_lamports: 10_000,
        node_stakes: vec![100],
        validator_configs: vec![validator_config],
        ..ClusterConfig::default()
    };
    let mut cluster = LocalCluster::new(&config);
    let client = cluster
        .get_validator_client(&cluster.entry_point_info.id)
        .unwrap();
    loop {
        let last_slot = client
            .get_slot_with_commitment(CommitmentConfig::recent())
            .expect("Couldn't get slot");
        if last_slot > 4 * VOTE_THRESHOLD_DEPTH as u64 {
            break;
        }
        sleep(Duration::from_secs(1));
    }

    cluster.close_preserve_ledgers();
    let leader_pubkey = cluster.entry_point_info.id;
    let ledger_path = cluster.validators[&leader_pubkey].info.ledger_path.clone();
    let ledger = Blockstore::open(&ledger_path).unwrap();
    for i in 0..2 * VOTE_THRESHOLD_DEPTH {
        let meta = ledger.meta(i as u64).unwrap().unwrap();
        let parent = meta.parent_slot;
        let expected_parent = i.saturating_sub(1);
        assert_eq!(parent, expected_parent as u64);
    }
}

fn wait_for_next_snapshot(
    cluster: &LocalCluster,
    snapshot_package_output_path: &Path,
) -> (PathBuf, (Slot, Hash)) {
    // Get slot after which this was generated
    let client = cluster
        .get_validator_client(&cluster.entry_point_info.id)
        .unwrap();
    let last_slot = client
        .get_slot_with_commitment(CommitmentConfig::recent())
        .expect("Couldn't get slot");

    // Wait for a snapshot for a bank >= last_slot to be made so we know that the snapshot
    // must include the transactions just pushed
    trace!(
        "Waiting for snapshot archive to be generated with slot > {}",
        last_slot
    );
    loop {
        if let Some((filename, (slot, hash, _))) =
            snapshot_utils::get_highest_snapshot_archive_path(snapshot_package_output_path)
        {
            trace!("snapshot for slot {} exists", slot);
            if slot >= last_slot {
                return (filename, (slot, hash));
            }
            trace!("snapshot slot {} < last_slot {}", slot, last_slot);
        }
        sleep(Duration::from_millis(5000));
    }
}

fn farf_dir() -> PathBuf {
    std::env::var("FARF_DIR")
        .unwrap_or_else(|_| "farf".to_string())
        .into()
}

fn generate_account_paths(num_account_paths: usize) -> (Vec<TempDir>, Vec<PathBuf>) {
    let account_storage_dirs: Vec<TempDir> = (0..num_account_paths)
        .map(|_| tempfile::tempdir_in(farf_dir()).unwrap())
        .collect();
    let account_storage_paths: Vec<_> = account_storage_dirs
        .iter()
        .map(|a| a.path().to_path_buf())
        .collect();
    (account_storage_dirs, account_storage_paths)
}

struct SnapshotValidatorConfig {
    _snapshot_dir: TempDir,
    snapshot_output_path: TempDir,
    account_storage_dirs: Vec<TempDir>,
    validator_config: ValidatorConfig,
}

fn setup_snapshot_validator_config(
    snapshot_interval_slots: u64,
    num_account_paths: usize,
) -> SnapshotValidatorConfig {
    // Create the snapshot config
    let snapshot_dir = tempfile::tempdir_in(farf_dir()).unwrap();
    let snapshot_output_path = tempfile::tempdir_in(farf_dir()).unwrap();
    let snapshot_config = SnapshotConfig {
        snapshot_interval_slots,
        snapshot_package_output_path: PathBuf::from(snapshot_output_path.path()),
        snapshot_path: PathBuf::from(snapshot_dir.path()),
        compression: CompressionType::Bzip2,
        snapshot_version: snapshot_utils::SnapshotVersion::default(),
    };

    // Create the account paths
    let (account_storage_dirs, account_storage_paths) = generate_account_paths(num_account_paths);

    // Create the validator config
    let mut validator_config = ValidatorConfig::default();
    validator_config.rpc_config.enable_validator_exit = true;
    validator_config.snapshot_config = Some(snapshot_config);
    validator_config.account_paths = account_storage_paths;
    validator_config.accounts_hash_interval_slots = snapshot_interval_slots;

    SnapshotValidatorConfig {
        _snapshot_dir: snapshot_dir,
        snapshot_output_path,
        account_storage_dirs,
        validator_config,
    }
}