gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Bolster test_fullnode_rotate() checks

Browse Source
This commit is contained in:
Michael Vines
2019-02-13 10:48:53 -08:00
parent 6187779d10
commit b03636dc33
1 changed files with 117 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
tests/multinode.rs
View File

@ -20,7 +20,7 @@ use solana_sdk::timing::duration_as_s;
use std::collections::{HashSet, VecDeque}; use std::collections::{HashSet, VecDeque};
use std::env; use std::env;
use std::fs::remove_dir_all; use std::fs::remove_dir_all;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::mpsc::channel; use std::sync::mpsc::channel;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::thread::{sleep, Builder}; use std::thread::{sleep, Builder};
@ -1806,7 +1806,9 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
let validator = Node::new_localhost_with_pubkey(validator_keypair.pubkey()); let validator = Node::new_localhost_with_pubkey(validator_keypair.pubkey());
info!("leader id: {}", leader_keypair.pubkey()); info!("leader id: {}", leader_keypair.pubkey());
info!("validator id: {}", validator_keypair.pubkey()); if include_validator {
info!("validator id: {}", validator_keypair.pubkey());
}
// Make a common mint and a genesis entry for both leader + validator ledgers // Make a common mint and a genesis entry for both leader + validator ledgers
let ( let (
@ -1848,8 +1850,8 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
last_entry_id = entries.last().unwrap().id; last_entry_id = entries.last().unwrap().id;
} }
let mut tick_height_of_next_rotation = ticks_per_slot; let mut leader_tick_height_of_next_rotation = ticks_per_slot;
let mut should_be_leader = true; let mut leader_should_be_leader = true;
if fullnode_config.leader_scheduler_config.ticks_per_slot == 1 { if fullnode_config.leader_scheduler_config.ticks_per_slot == 1 {
// Add another tick to the ledger if the cluster has been configured for 1 tick_per_slot. // Add another tick to the ledger if the cluster has been configured for 1 tick_per_slot.
// The "pseudo-tick" entry0 currently added by bank::process_ledger cannot be rotated on // The "pseudo-tick" entry0 currently added by bank::process_ledger cannot be rotated on
@ -1859,12 +1861,22 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
entries.extend(tick); entries.extend(tick);
last_entry_id = entries.last().unwrap().id; last_entry_id = entries.last().unwrap().id;
tick_height_of_next_rotation += 2; leader_tick_height_of_next_rotation += 2;
if include_validator { if include_validator {
should_be_leader = false; leader_should_be_leader = false;
} }
} }
let validator_tick_height_of_next_rotation = Arc::new(AtomicUsize::new(
leader_tick_height_of_next_rotation as usize,
));
let leader_tick_height_of_next_rotation = Arc::new(AtomicUsize::new(
leader_tick_height_of_next_rotation as usize,
));
let validator_should_be_leader = Arc::new(AtomicBool::new(!leader_should_be_leader));
let leader_should_be_leader = Arc::new(AtomicBool::new(leader_should_be_leader));
// Write additional ledger entires // Write additional ledger entires
{ {
trace!("last_entry_id: {:?}", last_entry_id); trace!("last_entry_id: {:?}", last_entry_id);
@ -1903,37 +1915,43 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
); );
let (validator_rotation_sender, validator_rotation_receiver) = channel(); let (validator_rotation_sender, validator_rotation_receiver) = channel();
node_exits.push(validator_fullnode.run(Some(validator_rotation_sender)));
let mut validator_tick_height_of_next_rotation = tick_height_of_next_rotation; monitor_threads.push({
let mut validator_should_be_leader = !should_be_leader; let validator_should_be_leader = validator_should_be_leader.clone();
let monitor_thread = Builder::new() let validator_tick_height_of_next_rotation =
.name("validator_rotation_receiver".to_string()) validator_tick_height_of_next_rotation.clone();
.spawn(move || loop { Builder::new()
match validator_rotation_receiver.recv() { .name("validator_rotation_receiver".to_string())
Ok((rotation_type, tick_height)) => { .spawn(move || loop {
info!( match validator_rotation_receiver.recv() {
"validator rotation event {:?} at tick_height={}", Ok((rotation_type, tick_height)) => {
rotation_type, tick_height info!(
); "validator rotation event {:?} at tick_height={}",
info!("validator should be leader? {}", validator_should_be_leader); rotation_type, tick_height
assert_eq!(validator_tick_height_of_next_rotation, tick_height); );
assert_eq!( let should_be_leader =
rotation_type, validator_should_be_leader.fetch_xor(true, Ordering::Relaxed);
if validator_should_be_leader { info!("validator should be leader? {}", should_be_leader);
FullnodeReturnType::LeaderToValidatorRotation assert_eq!(
} else { tick_height as usize,
FullnodeReturnType::ValidatorToLeaderRotation validator_tick_height_of_next_rotation
} .fetch_add(ticks_per_slot as usize, Ordering::Relaxed)
); );
validator_should_be_leader = !validator_should_be_leader; assert_eq!(
validator_tick_height_of_next_rotation += ticks_per_slot; rotation_type,
} if should_be_leader {
Err(_) => break, FullnodeReturnType::LeaderToValidatorRotation
}; } else {
}) FullnodeReturnType::ValidatorToLeaderRotation
.unwrap(); }
monitor_threads.push(monitor_thread); );
}
Err(_) => break,
};
})
.unwrap()
});
node_exits.push(validator_fullnode.run(Some(validator_rotation_sender)));
} }
let (leader_rotation_sender, leader_rotation_receiver) = channel(); let (leader_rotation_sender, leader_rotation_receiver) = channel();
@ -1946,6 +1964,41 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
&fullnode_config, &fullnode_config,
); );
monitor_threads.push({
let leader_should_be_leader = leader_should_be_leader.clone();
let leader_tick_height_of_next_rotation = leader_tick_height_of_next_rotation.clone();
Builder::new()
.name("leader_rotation_receiver".to_string())
.spawn(move || loop {
match leader_rotation_receiver.recv() {
Ok((rotation_type, tick_height)) => {
info!(
"leader rotation event {:?} at tick_height={}",
rotation_type, tick_height
);
let should_be_leader =
leader_should_be_leader.fetch_xor(true, Ordering::Relaxed);
info!("leader should be leader? {}", should_be_leader);
assert_eq!(
tick_height as usize,
leader_tick_height_of_next_rotation
.fetch_add(ticks_per_slot as usize, Ordering::Relaxed)
);
assert_eq!(
rotation_type,
if should_be_leader {
FullnodeReturnType::LeaderToValidatorRotation
} else {
FullnodeReturnType::ValidatorToLeaderRotation
}
);
}
Err(_) => break,
};
})
.unwrap()
});
node_exits.push(leader_fullnode.run(Some(leader_rotation_sender))); node_exits.push(leader_fullnode.run(Some(leader_rotation_sender)));
converge(&leader_info, node_exits.len()); converge(&leader_info, node_exits.len());
@ -1954,33 +2007,17 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
poll_gossip_for_leader(leader_info.gossip, Some(5)).unwrap() poll_gossip_for_leader(leader_info.gossip, Some(5)).unwrap()
); );
const N: usize = 10; let max_tick_height = 16;
for i in 1..=N { while leader_tick_height_of_next_rotation.load(Ordering::Relaxed) < max_tick_height {
info!("waiting for rotation #{}/{}", i, N); trace!("waiting for leader to reach max tick height...");
match leader_rotation_receiver.recv_timeout(Duration::from_secs(5)) { sleep(Duration::from_millis(100));
Ok((rotation_type, tick_height)) => { }
info!(
"leader rotation event {:?} at tick_height={}", if include_validator {
rotation_type, tick_height while validator_tick_height_of_next_rotation.load(Ordering::Relaxed) < max_tick_height {
); trace!("waiting for validator to reach max tick height...");
assert_eq!(tick_height_of_next_rotation, tick_height); sleep(Duration::from_millis(100));
if include_validator { }
assert_eq!(
rotation_type,
if should_be_leader {
FullnodeReturnType::LeaderToValidatorRotation
} else {
FullnodeReturnType::ValidatorToLeaderRotation
}
);
should_be_leader = !should_be_leader;
} else {
assert_eq!(rotation_type, FullnodeReturnType::LeaderToLeaderRotation);
}
tick_height_of_next_rotation += ticks_per_slot;
}
err => panic!("Unexpected response: {:?}", err),
};
} }
info!("Shutting down"); info!("Shutting down");
@ -1996,6 +2033,23 @@ fn test_fullnode_rotate(ticks_per_slot: u64, slots_per_epoch: u64, include_valid
Blocktree::destroy(&path).unwrap(); Blocktree::destroy(&path).unwrap();
remove_dir_all(path).unwrap(); remove_dir_all(path).unwrap();
} }
trace!(
"final validator_tick_height_of_next_rotation: {}",
validator_tick_height_of_next_rotation.load(Ordering::Relaxed)
);
trace!(
"final leader_tick_height_of_next_rotation: {}",
leader_tick_height_of_next_rotation.load(Ordering::Relaxed)
);
trace!(
"final leader_should_be_leader: {}",
leader_should_be_leader.load(Ordering::Relaxed)
);
trace!(
"final validator_should_be_leader: {}",
validator_should_be_leader.load(Ordering::Relaxed)
);
} }
#[test] #[test]