gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor fullnode rotation test (#3015)

Browse Source
This commit is contained in:
Pankaj Garg
2019-02-28 15:53:09 -08:00
committed by GitHub
parent 30bccc0c68
commit d5f0e49535
2 changed files with 332 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/replay_stage.rs
View File

@ -377,6 +377,11 @@ impl ReplayStage {
cluster_info.write().unwrap().set_leader(leader_id);
}
trace!(
"node {:?} scheduled as leader for slot {}",
leader_id,
next_slot
);
// Always send rotation signal so that other services like
// RPC can be made aware of last slot's bank
to_leader_sender

409
tests/multinode.rs
View File

@ -6,6 +6,8 @@ use solana::blob_fetch_stage::BlobFetchStage;
use solana::blocktree::{create_new_tmp_ledger, tmp_copy_blocktree, Blocktree};
use solana::client::mk_client;
use solana::cluster_info::{Node, NodeInfo};
use solana::contact_info::ContactInfo;
use solana::entry::next_entry_mut;
use solana::entry::{reconstruct_entries_from_blobs, Entry};
use solana::fullnode::make_active_set_entries;
use solana::fullnode::{new_banks_from_blocktree, Fullnode, FullnodeConfig, FullnodeReturnType};
@ -16,14 +18,17 @@ use solana::service::Service;
use solana::thin_client::{poll_gossip_for_leader, retry_get_balance};
use solana::voting_keypair::VotingKeypair;
use solana_sdk::genesis_block::GenesisBlock;
use solana_sdk::hash::Hash;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, KeypairUtil};
use solana_sdk::system_transaction::SystemTransaction;
use solana_sdk::timing::duration_as_s;
use solana_sdk::vote_transaction::VoteTransaction;
use std::collections::{HashSet, VecDeque};
use std::env;
use std::fs::remove_dir_all;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::Receiver;
use std::sync::mpsc::{channel, sync_channel, TryRecvError};
use std::sync::{Arc, RwLock};
use std::thread::{sleep, Builder};
@ -1572,6 +1577,280 @@ fn retry_send_tx_and_retry_get_balance(
None
}
fn new_fullnode() -> (Arc<Keypair>, Node, ContactInfo) {
let keypair = Arc::new(Keypair::new());
let node = Node::new_localhost_with_pubkey(keypair.pubkey());
let node_info = node.info.clone();
(keypair, node, node_info)
}
fn new_genesis_block(
leader: Pubkey,
ticks_per_slot: u64,
slots_per_epoch: u64,
) -> (GenesisBlock, Keypair) {
let (mut genesis_block, mint_keypair) =
GenesisBlock::new_with_leader(1_000_000_000_000_000_000, leader, 100);
genesis_block.ticks_per_slot = ticks_per_slot;
genesis_block.slots_per_epoch = slots_per_epoch;
(genesis_block, mint_keypair)
}
fn fund_fullnode(
from: &Arc<Keypair>,
to: Pubkey,
tokens: u64,
last_tick: &Hash,
last_hash: &mut Hash,
entries: &mut Vec<Entry>,
) {
let transfer_tx = SystemTransaction::new_account(from, to, tokens, *last_tick, 0);
let transfer_entry = next_entry_mut(last_hash, 1, vec![transfer_tx]);
entries.extend(vec![transfer_entry]);
}
fn stake_fullnode(
node: &Arc<Keypair>,
stake: u64,
last_tick: &Hash,
last_hash: &mut Hash,
entries: &mut Vec<Entry>,
) -> VotingKeypair {
// Create and register a vote account for active_keypair
let voting_keypair = VotingKeypair::new_local(node);
let vote_account_id = voting_keypair.pubkey();
let new_vote_account_tx =
VoteTransaction::new_account(node, vote_account_id, *last_tick, stake, 0);
let new_vote_account_entry = next_entry_mut(last_hash, 1, vec![new_vote_account_tx]);
/*
let vote_tx = VoteTransaction::new_vote(&voting_keypair, 1, *last_tick, 0);
let vote_entry = next_entry_mut(last_hash, 1, vec![vote_tx]);
entries.extend(vec![new_vote_account_entry, vote_entry]);
*/
entries.extend(vec![new_vote_account_entry]);
voting_keypair
}
fn add_tick(last_id: &mut Hash, entries: &mut Vec<Entry>) -> Hash {
let tick = solana::entry::create_ticks(1, *last_id);
*last_id = tick[0].id;
entries.extend(tick);
*last_id
}
fn start_fullnode(
node: Node,
kp: Arc<Keypair>,
v_kp: VotingKeypair,
ledger: &str,
leader: Option<&NodeInfo>,
config: &FullnodeConfig,
) -> (impl FnOnce(), Receiver<(FullnodeReturnType, u64)>) {
let (rotation_sender, rotation_receiver) = channel();
let fullnode = Fullnode::new(node, &kp, ledger, v_kp, leader, &config);
(fullnode.run(Some(rotation_sender)), rotation_receiver)
}
//fn stop_fullnode() {}
fn new_non_bs_leader_fullnode(
mint: &Arc<Keypair>,
last_tick: &mut Hash,
mut last_id: &mut Hash,
entries: &mut Vec<Entry>,
) -> (Node, Arc<Keypair>, VotingKeypair) {
// Create the node information
let (node_keypair, node, _) = new_fullnode();
let voting = {
fund_fullnode(
mint,
node_keypair.pubkey(),
50,
&last_tick,
&mut last_id,
entries,
);
let voting = stake_fullnode(&node_keypair, 10, &last_tick, &mut last_id, entries);
*last_tick = add_tick(last_id, entries);
voting
};
(node, node_keypair, voting)
}
fn new_bs_leader_fullnode(
ticks_per_slot: u64,
slots_per_epoch: u64,
entries: &mut Vec<Entry>,
) -> (
Arc<Keypair>,
String,
Hash,
Hash,
(Node, Arc<Keypair>, VotingKeypair),
) {
// Create the node information
let (node_keypair, node, _) = new_fullnode();
let (genesis_block, mint_keypair) =
new_genesis_block(node_keypair.pubkey(), ticks_per_slot, slots_per_epoch);
let (ledger_path, mut last_id) = create_new_tmp_ledger!(&genesis_block);
let mut last_tick = add_tick(&mut last_id, entries);
let voting = stake_fullnode(&node_keypair, 20, &mut last_tick, &mut last_id, entries);
(
Arc::new(mint_keypair),
ledger_path,
last_tick,
last_id,
(node, node_keypair, voting),
)
}
#[test]
#[ignore]
fn test_fullnodes_bootup() {
let ticks_per_slot = 1;
let slots_per_epoch = 1;
solana_logger::setup();
// Create fullnode config, and set node scheduler policies
let fullnode_config = FullnodeConfig::default();
// let (tick_step_sender, tick_step_receiver) = sync_channel(1);
// fullnode_config.tick_config = PohServiceConfig::Step(tick_step_sender);
let mut entries = vec![];
let (mint, ledger, mut last_tick, mut last_id, leader) =
new_bs_leader_fullnode(ticks_per_slot, slots_per_epoch, &mut entries);
let leader_info = leader.0.info.clone();
let validator = new_non_bs_leader_fullnode(&mint, &mut last_tick, &mut last_id, &mut entries);
{
info!("Number of entries {}", entries.len());
trace!("last_id: {:?}", last_id);
trace!("last_tick: {:?}", last_tick);
trace!("entries: {:?}", entries);
let blocktree = Blocktree::open_config(&ledger, ticks_per_slot).unwrap();
blocktree.write_entries(1, 0, 0, &entries).unwrap();
}
// let validator_info = validator.0.info.clone();
let v_ledger = tmp_copy_blocktree!(&ledger);
let mut exits = vec![];
let (v_e, v_r_r) = start_fullnode(
validator.0,
validator.1,
validator.2,
&v_ledger,
Some(&leader_info),
&fullnode_config,
);
exits.push(v_e);
let (l_e, l_r_r) = start_fullnode(
leader.0,
leader.1,
leader.2,
&ledger,
None,
&fullnode_config,
);
exits.push(l_e);
converge(&leader_info, exits.len());
info!(
"found leader: {:?}",
poll_gossip_for_leader(leader_info.gossip, Some(5)).unwrap()
);
let mut leader_should_be_leader = true;
let mut validator_should_be_leader = false;
let mut leader_slot_height_of_next_rotation = 4;
let mut validator_slot_height_of_next_rotation = 4;
let max_slot_height = 8;
/*
let bob = Keypair::new().pubkey();
let mut client_last_id = solana_sdk::hash::Hash::default();
*/
while leader_slot_height_of_next_rotation < max_slot_height
&& validator_slot_height_of_next_rotation < max_slot_height
{
// Check for leader rotation
{
match l_r_r.try_recv() {
Ok((rotation_type, slot)) => {
if slot == 0 {
// Skip slot 0, as the nodes are not fully initialized in terms of leader scheduler
continue;
}
info!(
"leader rotation event {:?} at slot={} {}",
rotation_type, slot, leader_slot_height_of_next_rotation
);
info!("leader should be leader? {}", leader_should_be_leader);
assert_eq!(slot, leader_slot_height_of_next_rotation);
assert_eq!(
rotation_type,
if leader_should_be_leader {
FullnodeReturnType::LeaderToValidatorRotation
} else {
FullnodeReturnType::ValidatorToLeaderRotation
}
);
leader_should_be_leader = !leader_should_be_leader;
leader_slot_height_of_next_rotation += 1;
}
Err(TryRecvError::Empty) => {}
err => panic!(err),
}
}
// Check for validator rotation
match v_r_r.try_recv() {
Ok((rotation_type, slot)) => {
if slot == 0 {
// Skip slot 0, as the nodes are not fully initialized in terms of leader scheduler
continue;
}
info!(
"validator rotation event {:?} at slot={} {}",
rotation_type, slot, validator_slot_height_of_next_rotation
);
info!("validator should be leader? {}", validator_should_be_leader);
assert_eq!(slot, validator_slot_height_of_next_rotation);
assert_eq!(
rotation_type,
if validator_should_be_leader {
FullnodeReturnType::LeaderToValidatorRotation
} else {
FullnodeReturnType::ValidatorToLeaderRotation
}
);
validator_slot_height_of_next_rotation += 1;
validator_should_be_leader = !validator_should_be_leader;
}
Err(TryRecvError::Empty) => {}
err => panic!(err),
}
}
}
fn test_fullnode_rotate(
ticks_per_slot: u64,
slots_per_epoch: u64,
@ -1580,110 +1859,75 @@ fn test_fullnode_rotate(
) {
solana_logger::setup();
// Create the leader node information
let leader_keypair = Arc::new(Keypair::new());
let leader = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
let leader_info = leader.info.clone();
let mut leader_should_be_leader = true;
let mut leader_slot_height_of_next_rotation = 2;
// Create fullnode config, and set leader scheduler policies
// Create fullnode config, and set node scheduler policies
let mut fullnode_config = FullnodeConfig::default();
let (tick_step_sender, tick_step_receiver) = sync_channel(1);
fullnode_config.tick_config = PohServiceConfig::Step(tick_step_sender);
// Note: when debugging failures in this test, disabling voting can help keep the log noise
// down by removing the extra vote transactions
/*
fullnode_config.voting_disabled = true;
*/
// Create the Genesis block using leader's keypair
let (mut genesis_block, mint_keypair) =
GenesisBlock::new_with_leader(1_000_000_000_000_000_000, leader_keypair.pubkey(), 123);
genesis_block.ticks_per_slot = ticks_per_slot;
genesis_block.slots_per_epoch = slots_per_epoch;
// Make a common mint and a genesis entry for both leader + validator ledgers
let (leader_ledger_path, mut last_id) = create_new_tmp_ledger!(&genesis_block);
let mut ledger_paths = Vec::new();
ledger_paths.push(leader_ledger_path.clone());
info!("ledger is {}", leader_ledger_path);
let mut entries = vec![];
// Create the validator node information
let validator_keypair = Arc::new(Keypair::new());
let validator = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
let (mint, ledger, mut last_tick, mut last_id, leader) =
new_bs_leader_fullnode(ticks_per_slot, slots_per_epoch, &mut entries);
let mut leader_slot_height_of_next_rotation = 1;
let leader_info = leader.0.info.clone();
if ticks_per_slot == 1 {
// Add another tick to the ledger if the cluster has been configured for 1 ticks_per_slot.
// The "pseudo-tick" entry0 currently added by bank::process_ledger cannot be rotated on
// since it has no last id (so at 1 ticks_per_slot rotation must start at a tick_height of
// 2)
let tick = solana::entry::create_ticks(1, last_id);
last_id = tick[0].id;
entries.extend(tick);
let validator = if include_validator {
leader_slot_height_of_next_rotation += 1;
}
Some(new_non_bs_leader_fullnode(
&mint,
&mut last_tick,
&mut last_id,
&mut entries,
))
} else {
None
};
// Setup the cluster with a single node
if include_validator {
// Add validator vote on tick height 1
let (active_set_entries, _) = make_active_set_entries(
&validator_keypair,
&mint_keypair,
100,
1,
&last_id,
ticks_per_slot,
);
last_id = active_set_entries.last().unwrap().id;
entries.extend(active_set_entries);
leader_slot_height_of_next_rotation += 1;
}
// Write additional ledger entries
{
trace!("last_id: {:?}", last_id);
trace!("last_tick: {:?}", last_tick);
trace!("entries: {:?}", entries);
let blocktree = Blocktree::open_config(&leader_ledger_path, ticks_per_slot).unwrap();
let blocktree = Blocktree::open_config(&ledger, ticks_per_slot).unwrap();
blocktree.write_entries(1, 0, 0, &entries).unwrap();
}
// Start up the node(s)
let mut ledger_paths = Vec::new();
ledger_paths.push(ledger.clone());
info!("ledger is {}", ledger);
let v_ledger = tmp_copy_blocktree!(&ledger);
ledger_paths.push(v_ledger.clone());
let mut node_exits = vec![];
let (validator_rotation_sender, validator_rotation_receiver) = channel();
if include_validator {
let validator_ledger_path = tmp_copy_blocktree!(&leader_ledger_path);
ledger_paths.push(validator_ledger_path.clone());
let validator_fullnode = Fullnode::new(
validator,
&validator_keypair,
&validator_ledger_path,
VotingKeypair::new_local(&validator_keypair),
let validator_rotation_receiver = if let Some(node) = validator {
let (v_e, v_r_r) = start_fullnode(
node.0,
node.1,
node.2,
&v_ledger,
Some(&leader_info),
&fullnode_config,
);
node_exits.push(v_e);
v_r_r
} else {
channel().1
};
node_exits.push(validator_fullnode.run(Some(validator_rotation_sender)));
}
let (leader_rotation_sender, leader_rotation_receiver) = channel();
let leader_fullnode = Fullnode::new(
leader,
&leader_keypair,
&leader_ledger_path,
VotingKeypair::new_local(&leader_keypair),
let (l_e, leader_rotation_receiver) = start_fullnode(
leader.0,
leader.1,
leader.2,
&ledger,
None,
&fullnode_config,
);
node_exits.push(leader_fullnode.run(Some(leader_rotation_sender)));
node_exits.push(l_e);
converge(&leader_info, node_exits.len());
info!(
@ -1708,11 +1952,14 @@ fn test_fullnode_rotate(
{
match leader_rotation_receiver.try_recv() {
Ok((rotation_type, slot)) => {
if slot == 0 {
if slot < leader_slot_height_of_next_rotation {
// Skip slot 0, as the nodes are not fully initialized in terms of leader scheduler
continue;
}
info!("leader rotation event {:?} at slot={}", rotation_type, slot);
info!(
"leader rotation event {:?} at slot={} {}",
rotation_type, slot, leader_slot_height_of_next_rotation
);
info!("leader should be leader? {}", leader_should_be_leader);
assert_eq!(slot, leader_slot_height_of_next_rotation);
if include_validator {
@ -1739,7 +1986,7 @@ fn test_fullnode_rotate(
if include_validator {
match validator_rotation_receiver.try_recv() {
Ok((rotation_type, slot)) => {
if slot == 0 {
if slot < validator_slot_height_of_next_rotation {
// Skip slot 0, as the nodes are not fully initialized in terms of leader scheduler
continue;
}
@ -1774,9 +2021,7 @@ fn test_fullnode_rotate(
info!("Transferring 500 tokens, last_id={:?}", client_last_id);
expected_bob_balance += 500;
let signature = client
.transfer(500, &mint_keypair, bob, &client_last_id)
.unwrap();
let signature = client.transfer(500, &mint, bob, &client_last_id).unwrap();
debug!("transfer send, signature is {:?}", signature);
for _ in 0..30 {
if client.poll_for_signature(&signature).is_err() {