2018-09-05 07:56:26 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								#!/bin/bash -ex
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								cd "$(dirname "$0")"/../..
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								deployMethod="$1"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								nodeType="$2"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								publicNetwork="$3"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								entrypointIp="$4"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								numNodes="$5"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								RUST_LOG="$6"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								[[ -n $deployMethod ]] || exit
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								[[ -n $nodeType ]] || exit
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								[[ -n $publicNetwork ]] || exit
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								[[ -n $entrypointIp ]] || exit
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								[[ -n $numNodes ]] || exit
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								cat > deployConfig <<EOF
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								deployMethod="$deployMethod"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								entrypointIp="$entrypointIp"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								numNodes="$numNodes"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								EOF
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								source net/common.sh
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								loadConfigFile
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								scripts/install-earlyoom.sh
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								if [[ $publicNetwork = true ]]; then
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  setupArgs="-p"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								else
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  setupArgs="-l"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								fi
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								case $deployMethod in
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								snap)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  SECONDS=0
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  sudo snap install solana.snap --devmode --dangerous
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  commonNodeConfig="\
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    leader-ip=$entrypointIp \
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 15:16:25 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    default-metrics-rate=1 \
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    metrics-config=$SOLANA_METRICS_CONFIG \
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    rust-log=$RUST_LOG \
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    setup-args=$setupArgs \
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  "
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  if [[ -e /dev/nvidia0 ]]; then
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    commonNodeConfig="$commonNodeConfig enable-cuda=1"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  fi
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  if [[ $nodeType = leader ]]; then
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    nodeConfig="mode=leader+drone $commonNodeConfig"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-05 08:51:40 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    ln -sf -T /var/snap/solana/current/leader/current leader.log
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ln -sf -T /var/snap/solana/current/drone/current drone.log
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  else
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    nodeConfig="mode=validator $commonNodeConfig"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-05 08:51:40 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    ln -sf -T /var/snap/solana/current/validator/current validator.log
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  fi
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  logmarker="solana deploy $(date)/$RANDOM"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  logger "$logmarker"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  # shellcheck disable=SC2086 # Don't want to double quote "$nodeConfig"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  sudo snap set solana $nodeConfig
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  snap info solana
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  sudo snap get solana
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  echo Slight delay to get more syslog output
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  sleep 2
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  sudo grep -Pzo "$logmarker(.|\\n)*" /var/log/syslog
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  echo "Succeeded in ${SECONDS} seconds"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ;;
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								local)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  PATH="$HOME"/.cargo/bin:"$PATH"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  export USE_INSTALL=1
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  export RUST_LOG
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 15:16:25 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  export SOLANA_DEFAULT_METRICS_RATE=1
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ./fetch-perf-libs.sh
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 23:15:39 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  export LD_LIBRARY_PATH="$PWD/target/perf-libs:$LD_LIBRARY_PATH"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  scripts/oom-monitor.sh  > oom-monitor.log 2>&1 &
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  case $nodeType in
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  leader)
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    ./multinode-demo/setup.sh -t leader $setupArgs
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ./multinode-demo/drone.sh > drone.log 2>&1 &
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ./multinode-demo/leader.sh > leader.log 2>&1 &
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ;;
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  validator)
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    ./multinode-demo/setup.sh -t validator $setupArgs
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-05 07:56:26 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp:8001" >validator.log 2>&1 &
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ;;
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  *)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    echo "Error: unknown node type: $nodeType"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    exit 1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ;;
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  esac
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ;;
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								*)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  echo "Unknown deployment method: $deployMethod"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  exit 1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								esac
							 |