From 4ff4fb6c38b6eb385c31800ac869eb860b531e4e Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Sun, 17 Feb 2019 09:48:27 -0800 Subject: [PATCH] Add support for an API node that hosts the block explorer --- net/common.sh | 2 ++ net/gce.sh | 55 ++++++++++++++++++++++++++++++++------- net/net.sh | 23 +++++++++++----- net/remote/remote-node.sh | 31 ++++++++++++++++++---- net/ssh.sh | 9 +++++++ 5 files changed, 99 insertions(+), 21 deletions(-) diff --git a/net/common.sh b/net/common.sh index cbf2146c60..e241b3e9a6 100644 --- a/net/common.sh +++ b/net/common.sh @@ -29,6 +29,8 @@ fullnodeIpList=() fullnodeIpListPrivate=() clientIpList=() clientIpListPrivate=() +apiIpList=() +apiIpListPrivate=() leaderRotation= buildSshOptions() { diff --git a/net/gce.sh b/net/gce.sh index c3de8a7caf..9433bc894b 100755 --- a/net/gce.sh +++ b/net/gce.sh @@ -17,6 +17,7 @@ gce) bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType fullNodeMachineType=n1-standard-16 clientMachineType=n1-standard-16 + apiNodeMachineType=n1-standard-8 ;; ec2) # shellcheck source=net/scripts/ec2-provider.sh @@ -27,6 +28,7 @@ ec2) bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType fullNodeMachineType=m4.2xlarge clientMachineType=m4.2xlarge + apiNodeMachineType=m4.2xlarge ;; *) echo "Error: Unknown cloud provider: $cloudProvider" @@ -37,12 +39,13 @@ esac prefix=testnet-dev-${USER//[^A-Za-z0-9]/} additionalFullNodeCount=5 clientNodeCount=1 +apiNode=false fullNodeBootDiskSizeInGb=1000 clientBootDiskSizeInGb=75 publicNetwork=false enableGpu=false -bootstrapLeaderAddress= +customAddress= leaderRotation=true usage() { @@ -69,14 +72,17 @@ Manage testnet instances create-specific options: -n [number] - Number of additional fullnodes (default: $additionalFullNodeCount) -c [number] - Number of client nodes (default: $clientNodeCount) + -u - Include an API node (default: $apiNode) -P - Use public network IP addresses (default: $publicNetwork) -g - Enable GPU (default: $enableGpu) - -G - Enable GPU, and set count/type of GPUs to use (e.g $cpuBootstrapLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80) - -a [address] - Set the bootstreap fullnode's external IP address to this value. - For GCE, [address] is the "name" of the desired External - IP Address. - For EC2, [address] is the "allocation ID" of the desired - Elastic IP. + -G - Enable GPU, and set count/type of GPUs to use + (e.g $cpuBootstrapLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80) + -a [address] - Address to be be assigned to the API node if present, + otherwise the bootstrap fullnode. + * For GCE, [address] is the "name" of the desired External + IP Address. + * For EC2, [address] is the "allocation ID" of the desired + Elastic IP. -d [disk-type] - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE. -b - Disable leader rotation @@ -100,7 +106,7 @@ shift [[ $command = create || $command = config || $command = info || $command = delete ]] || usage "Invalid command: $command" -while getopts "h?p:Pn:c:z:gG:a:d:b" opt; do +while getopts "h?p:Pn:c:z:gG:a:d:bu" opt; do case $opt in h | \?) usage @@ -133,11 +139,14 @@ while getopts "h?p:Pn:c:z:gG:a:d:b" opt; do bootstrapLeaderMachineType="$OPTARG" ;; a) - bootstrapLeaderAddress=$OPTARG + customAddress=$OPTARG ;; d) bootDiskType=$OPTARG ;; + u) + apiNode=true + ;; *) usage "unhandled option: $opt" ;; @@ -327,6 +336,15 @@ EOF cloud_ForEachInstance waitForStartupComplete } + echo "apiIpList=()" >> "$configFile" + echo "apiIpListPrivate=()" >> "$configFile" + echo "Looking for api instances..." + cloud_FindInstances "$prefix-api" + [[ ${#instances[@]} -eq 0 ]] || { + cloud_ForEachInstance recordInstanceIp apiIpList + cloud_ForEachInstance waitForStartupComplete + } + echo "Wrote $configFile" $metricsWriteDatapoint "testnet-deploy net-config-complete=1" } @@ -382,6 +400,7 @@ Network composition: Bootstrap leader = $bootstrapLeaderMachineType (GPU=$enableGpu) Additional fullnodes = $additionalFullNodeCount x $fullNodeMachineType Client(s) = $clientNodeCount x $clientMachineType + API Node = $apiNode Leader rotation: $leaderRotation @@ -451,6 +470,12 @@ touch /.instance-startup-complete EOF + if $apiNode; then + apiNodeAddress=$customAddress + else + bootstrapLeaderAddress=$customAddress + fi + cloud_CreateInstances "$prefix" "$prefix-bootstrap-leader" 1 \ "$imageName" "$bootstrapLeaderMachineType" "$fullNodeBootDiskSizeInGb" \ "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" @@ -465,6 +490,12 @@ EOF "$startupScript" "" "$bootDiskType" fi + if $apiNode; then + cloud_CreateInstances "$prefix" "$prefix-api" "1" \ + "$imageName" "$apiNodeMachineType" "$fullNodeBootDiskSizeInGb" \ + "$startupScript" "$apiNodeAddress" "$bootDiskType" + fi + $metricsWriteDatapoint "testnet-deploy net-create-complete=1" prepareInstancesAndWriteConfigFile @@ -497,6 +528,12 @@ info) ipAddressPrivate=${clientIpListPrivate[$i]} printNode bench-tps "$ipAddress" "$ipAddressPrivate" done + + for i in $(seq 0 $(( ${#apiIpList[@]} - 1)) ); do + ipAddress=${apiIpList[$i]} + ipAddressPrivate=${apiIpListPrivate[$i]} + printNode api "$ipAddress" "$ipAddressPrivate" + done ;; *) usage "Unknown command: $command" diff --git a/net/net.sh b/net/net.sh index c31522ec58..5d3f8b477b 100755 --- a/net/net.sh +++ b/net/net.sh @@ -225,9 +225,10 @@ startBootstrapLeader() { startNode() { declare ipAddress=$1 + declare nodeType=$2 declare logFile="$netLogDir/fullnode-$ipAddress.log" - echo "--- Starting fullnode: $ipAddress" + echo "--- Starting $nodeType: $ipAddress" echo "start log: $logFile" ( set -x @@ -235,7 +236,7 @@ startNode() { ssh "${sshOptions[@]}" -n "$ipAddress" \ "./solana/net/remote/remote-node.sh \ $deployMethod \ - fullnode \ + $nodeType \ $publicNetwork \ $entrypointIp \ ${#fullnodeIpList[@]} \ @@ -347,8 +348,13 @@ start() { $metricsWriteDatapoint "testnet-deploy net-start-begin=1" fi - bootstrapLeader=true - for ipAddress in "${fullnodeIpList[@]}"; do + declare bootstrapLeader=true + declare nodeType=fullnode + for ipAddress in "${fullnodeIpList[@]}" - "${apiIpList[@]}"; do + if [[ $ipAddress = - ]]; then + nodeType=apinode + continue + fi if $updateNodes; then stopNode "$ipAddress" fi @@ -364,7 +370,7 @@ start() { pids=() loopCount=0 else - startNode "$ipAddress" + startNode "$ipAddress" $nodeType # Stagger additional node start time. If too many nodes start simultaneously # the bootstrap node gets more rsync requests from the additional nodes than @@ -431,7 +437,7 @@ start() { echo echo "+++ Deployment Successful" echo "Bootstrap leader deployment took $bootstrapNodeDeployTime seconds" - echo "Additional fullnode deployment (${#fullnodeIpList[@]} instances) took $additionalNodeDeployTime seconds" + echo "Additional fullnode deployment (${#fullnodeIpList[@]} full nodes, ${#apiIpList[@]} api nodes) took $additionalNodeDeployTime seconds" echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds" echo "Network start logs in $netLogDir:" ls -l "$netLogDir" @@ -466,7 +472,7 @@ stop() { SECONDS=0 $metricsWriteDatapoint "testnet-deploy net-stop-begin=1" - for ipAddress in "${fullnodeIpList[@]}" "${clientIpList[@]}"; do + for ipAddress in "${fullnodeIpList[@]}" "${apiIpList[@]}" "${clientIpList[@]}"; do stopNode "$ipAddress" done @@ -515,6 +521,9 @@ logs) for ipAddress in "${clientIpList[@]}"; do fetchRemoteLog "$ipAddress" client done + for ipAddress in "${apiIpList[@]}"; do + fetchRemoteLog "$ipAddress" fullnode + done ;; *) diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index c84aed714a..7581d00125 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -98,14 +98,21 @@ snap) echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" fi - if [[ $nodeType = bootstrap-leader ]]; then + case $nodeType in + bootstrap-leader) nodeConfig="mode=bootstrap-leader+drone $commonNodeConfig" ln -sf -T /var/snap/solana/current/bootstrap-leader/current fullnode.log ln -sf -T /var/snap/solana/current/drone/current drone.log - else + ;; + fullnode) nodeConfig="mode=fullnode $commonNodeConfig" ln -sf -T /var/snap/solana/current/fullnode/current fullnode.log - fi + ;; + *) + echo "Error: unknown node type: $nodeType" + exit 1 + ;; + esac logmarker="solana deploy $(date)/$RANDOM" logger "$logmarker" @@ -163,7 +170,7 @@ local|tar) ./multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation > bootstrap-leader.log 2>&1 & ln -sTf bootstrap-leader.log fullnode.log ;; - fullnode) + fullnode|apinode) net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/ if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-fullnode-cuda ]]; then @@ -171,11 +178,25 @@ local|tar) export SOLANA_CUDA=1 fi + args=("$maybeNoLeaderRotation") + if [[ $nodeType = apinode ]]; then + args+=(--entry-stream /tmp/solana-entry-stream.sock) + fi + set -x if [[ $skipSetup != true ]]; then ./multinode-demo/setup.sh -t fullnode $setupArgs + + if [[ $nodeType = apinode ]]; then + npm install @solana/blockexplorer + fi fi - ./multinode-demo/fullnode.sh $maybeNoLeaderRotation "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 & + + # Run blockexplorer as root so it can bind to port 80 + # shellcheck disable=SC2024 # "sudo doesn't affect redirects" warning does not apply + sudo npx solana-blockexplorer > blockexplorer.log 2>&1 & + + ./multinode-demo/fullnode.sh "${args[@]}" "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 & ;; *) echo "Error: unknown node type: $nodeType" diff --git a/net/ssh.sh b/net/ssh.sh index 48c8f0d53d..33156da10d 100755 --- a/net/ssh.sh +++ b/net/ssh.sh @@ -63,6 +63,15 @@ else done fi echo +echo API nodes: +if [[ ${#apiIpList[@]} -eq 0 ]]; then + echo " None" +else + for ipAddress in "${apiIpList[@]}"; do + printNode fullnode "$ipAddress" + done +fi +echo echo "Use |scp.sh| to transfer files to and from nodes" echo