2018-11-11 08:19:04 -08:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								#!/usr/bin/env bash
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								set -e
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-28 10:19:33 -07:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 09:21:03 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								cd "$(dirname "$0")"/../..
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								deployMethod="$1"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								entrypointIp="$2"
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								clientToRun="$3"
							 | 
						
					
						
							
								
									
										
										
										
											2019-07-31 10:13:30 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								if [[ -n $4 ]]; then
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  export RUST_LOG="$4"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								fi
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-23 15:13:29 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								benchTpsExtraArgs="$5"
							 | 
						
					
						
							
								
									
										
										
										
											2021-07-12 21:59:11 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								clientIndex="$6"
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-04 22:21:58 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 08:46:20 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								missing() {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  echo "Error: $1 not specified"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  exit 1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								[[ -n $deployMethod ]] || missing deployMethod
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								[[ -n $entrypointIp ]] || missing entrypointIp
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-28 10:19:33 -07:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								source net/common.sh
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								loadConfigFile
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								threadCount=$(nproc)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								if [[ $threadCount -gt 4 ]]; then
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  threadCount=4
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								fi
							 | 
						
					
						
							
								
									
										
										
										
											2018-08-28 10:19:33 -07:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								case $deployMethod in
							 | 
						
					
						
							
								
									
										
										
										
											2018-10-30 18:05:38 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								local|tar)
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  PATH="$HOME"/.cargo/bin:"$PATH"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  export USE_INSTALL=1
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-26 13:36:51 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/*" ~/.cargo/bin/
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  ;;
							 | 
						
					
						
							
								
									
										
										
										
											2019-07-22 21:38:26 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								skip)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ;;
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-03 18:15:55 -10:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								*)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  echo "Unknown deployment method: $deployMethod"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  exit 1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								esac
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								case $clientToRun in
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								solana-bench-tps)
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-11 18:47:35 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  net/scripts/rsync-retry.sh -vPrc \
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-19 08:31:22 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    "$entrypointIp":~/solana/config/bench-tps"$clientIndex".yml ./client-accounts.yml
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  clientCommand="\
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    solana-bench-tps \
							 | 
						
					
						
							
								
									
										
										
										
											2019-05-03 15:00:19 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      --entrypoint $entrypointIp:8001 \
							 | 
						
					
						
							
								
									
										
										
										
											2019-12-16 14:05:17 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      --faucet $entrypointIp:9900 \
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      --duration 7500 \
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      --sustained \
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								      --threads $threadCount \
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-23 15:13:29 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      $benchTpsExtraArgs \
							 | 
						
					
						
							
								
									
										
										
										
											2019-06-11 18:47:35 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								      --read-client-keys ./client-accounts.yml \
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  "
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ;;
							 | 
						
					
						
							
								
									
										
										
										
											2019-10-23 09:56:05 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								idle)
							 | 
						
					
						
							
								
									
										
										
										
											2020-01-31 12:14:05 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  # Add the faucet keypair to idle clients for convenience
							 | 
						
					
						
							
								
									
										
										
										
											2019-10-25 18:23:52 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  net/scripts/rsync-retry.sh -vPrc \
							 | 
						
					
						
							
								
									
										
										
										
											2020-03-14 16:49:28 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    "$entrypointIp":~/solana/config/faucet.json ~/solana/
							 | 
						
					
						
							
								
									
										
										
										
											2019-10-23 09:56:05 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  exit 0
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ;;
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								*)
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  echo "Unknown client name: $clientToRun"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  exit 1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								esac
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 20:21:41 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-10 12:30:06 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								cat > ~/solana/on-reboot <<EOF
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								#!/usr/bin/env bash
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								cd ~/solana
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								PATH="$HOME"/.cargo/bin:"$PATH"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								export USE_INSTALL=1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								echo "$(date) | $0 $*" >> client.log
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								(
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  sudo SOLANA_METRICS_CONFIG="$SOLANA_METRICS_CONFIG" scripts/oom-monitor.sh
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								) > oom-monitor.log 2>&1 &
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-19 08:31:22 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								echo \$! > oom-monitor.pid
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-10 12:30:06 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								scripts/fd-monitor.sh > fd-monitor.log 2>&1 &
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-19 08:31:22 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								echo \$! > fd-monitor.pid
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-10 12:30:06 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								scripts/net-stats.sh  > net-stats.log 2>&1 &
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-19 08:31:22 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								echo \$! > net-stats.pid
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-10 12:30:06 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								! tmux list-sessions || tmux kill-session
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								tmux new -s "$clientToRun" -d "
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 08:34:42 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  while true; do
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 20:50:46 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    echo === Client start: \$(date) | tee -a client.log
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 08:34:42 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    $metricsWriteDatapoint 'testnet-deploy client-begin=1'
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 20:50:46 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    echo '$ $clientCommand' | tee -a client.log
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 08:34:42 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    $clientCommand >> client.log 2>&1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    $metricsWriteDatapoint 'testnet-deploy client-complete=1'
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  done
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								"
							 | 
						
					
						
							
								
									
										
										
										
											2019-09-10 12:30:06 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								EOF
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								chmod +x ~/solana/on-reboot
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								echo "@reboot ~/solana/on-reboot" | crontab -
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								~/solana/on-reboot
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-09-07 08:34:42 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								sleep 1
							 | 
						
					
						
							
								
									
										
										
										
											2019-04-19 09:56:01 -07:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								tmux capture-pane -t "$clientToRun" -p -S -100
							 |