gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Start drone on remote leader node

Browse Source 
- Also, enables CUDA for leader node
This commit is contained in:
Pankaj Garg
2018-07-03 10:40:11 -07:00
committed by Greg Fitzgerald
parent 87156e1364
commit 5bc133985b
1 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
multinode-demo/start_nodes.sh
View File

@ -23,18 +23,24 @@ if [[ -z "$remote_user" ]]; then
usage usage
fi fi
echo "Build started at $(date)"
SECONDS=0
# Build and install locally # Build and install locally
PATH="$HOME"/.cargo/bin:"$PATH" PATH="$HOME"/.cargo/bin:"$PATH"
cargo install --force cargo install --force
echo "Build took $SECONDS seconds"
ip_addr_array=() ip_addr_array=()
# Get IP address array # Get IP address array
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "$ip_addr_file" source "$ip_addr_file"
# shellcheck disable=SC2089,SC2016 # shellcheck disable=SC2089,SC2016
ssh_command_prefix='export PATH="$HOME/.cargo/bin:$PATH"; cd solana; USE_INSTALL=1 ./multinode-demo/' ssh_command_prefix='export PATH="$HOME/.cargo/bin:$PATH"; cd solana; USE_INSTALL=1'
echo "Deployment started at $(date)"
SECONDS=0
count=0 count=0
leader= leader=
for ip_addr in "${ip_addr_array[@]}"; do for ip_addr in "${ip_addr_array[@]}"; do
@ -43,7 +49,7 @@ for ip_addr in "${ip_addr_array[@]}"; do
ssh-keygen -R "$ip_addr" ssh-keygen -R "$ip_addr"
ssh-keyscan "$ip_addr" >>~/.ssh/known_hosts ssh-keyscan "$ip_addr" >>~/.ssh/known_hosts
ssh "$remote_user@$ip_addr" 'mkdir -p ~/.ssh ~/solana ~/.cargo/bin' ssh -n -f "$remote_user@$ip_addr" 'mkdir -p ~/.ssh ~/solana ~/.cargo/bin'
# Killing sshguard for now. TODO: Find a better solution # Killing sshguard for now. TODO: Find a better solution
# sshguard is blacklisting IP address after ssh-keyscan and ssh login attempts # sshguard is blacklisting IP address after ssh-keyscan and ssh login attempts
@ -61,35 +67,41 @@ for ip_addr in "${ip_addr_array[@]}"; do
fi fi
# Stop current nodes # Stop current nodes
ssh "$remote_user@$ip_addr" 'pkill -9 solana-fullnode' ssh "$remote_user@$ip_addr" 'pkill -9 solana-'
ssh "$remote_user@$ip_addr" 'pkill -9 solana-client-demo'
if [[ -n $leader ]]; then if [[ -n $leader ]]; then
echo "Adding known hosts for $ip_addr" echo "Adding known hosts for $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "ssh-keygen -R $leader" ssh -n -f "$remote_user@$ip_addr" "ssh-keygen -R $leader"
ssh -n -f "$remote_user@$ip_addr" "ssh-keyscan $leader >> ~/.ssh/known_hosts" ssh -n -f "$remote_user@$ip_addr" "ssh-keyscan $leader >> ~/.ssh/known_hosts"
ssh "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/.cargo/bin/solana* ~/.cargo/bin/" ssh -n -f "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/.cargo/bin/solana* ~/.cargo/bin/"
ssh "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/solana/multinode-demo ~/solana/" ssh -n -f "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/solana/multinode-demo ~/solana/"
ssh -n -f "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/solana/fetch-perf-libs.sh ~/solana/"
else else
# Deploy build and scripts to remote node # Deploy build and scripts to remote node
rsync -vPrz ~/.cargo/bin/solana* "$remote_user@$ip_addr":~/.cargo/bin/ rsync -vPrz ~/.cargo/bin/solana* "$remote_user@$ip_addr":~/.cargo/bin/
rsync -vPrz ./multinode-demo "$remote_user@$ip_addr":~/solana/ rsync -vPrz ./multinode-demo "$remote_user@$ip_addr":~/solana/
rsync -vPrz ./fetch-perf-libs.sh "$remote_user@$ip_addr":~/solana/
fi fi
# Run setup # Run setup
ssh "$remote_user@$ip_addr" "$ssh_command_prefix"'setup.sh -p "$ip_addr"' ssh "$remote_user@$ip_addr" "$ssh_command_prefix"' ./multinode-demo/setup.sh -p "$ip_addr"'
if ((!count)); then if ((!count)); then
# Start the leader on the first node # Start the leader on the first node
echo "Starting leader node $ip_addr" echo "Starting leader node $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"'leader.sh > leader.log 2>&1' ssh -n -f "$remote_user@$ip_addr" 'cd solana; ./fetch-perf-libs.sh'
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"' SOLANA_CUDA=1 ./multinode-demo/leader.sh > leader.log 2>&1'
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"' ./multinode-demo/drone.sh > drone.log 2>&1'
leader=${ip_addr_array[0]} leader=${ip_addr_array[0]}
else else
# Start validator on all other nodes # Start validator on all other nodes
echo "Starting validator node $ip_addr" echo "Starting validator node $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix""validator.sh $remote_user@$leader:~/solana $leader > validator.log 2>&1" ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"" ./multinode-demo/validator.sh $remote_user@$leader:~/solana $leader > validator.log 2>&1"
fi fi
((count++)) ((count++))
done done
echo "Deployment finished at $(date)"
echo "Deployment took $SECONDS seconds"