Add support for an API node that hosts the block explorer

net/common.sh

@@ -29,6 +29,8 @@ fullnodeIpList=()
 fullnodeIpListPrivate=()
 clientIpList=()
 clientIpListPrivate=()
+apiIpList=()
+apiIpListPrivate=()
 leaderRotation=
 
 buildSshOptions() {

net/gce.sh

@@ -17,6 +17,7 @@ gce)
   bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType
   fullNodeMachineType=n1-standard-16
   clientMachineType=n1-standard-16
+  apiNodeMachineType=n1-standard-8
   ;;
 ec2)
   # shellcheck source=net/scripts/ec2-provider.sh
@@ -27,6 +28,7 @@ ec2)
   bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType
   fullNodeMachineType=m4.2xlarge
   clientMachineType=m4.2xlarge
+  apiNodeMachineType=m4.2xlarge
   ;;
 *)
   echo "Error: Unknown cloud provider: $cloudProvider"
@@ -37,12 +39,13 @@ esac
 prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
 additionalFullNodeCount=5
 clientNodeCount=1
+apiNode=false
 fullNodeBootDiskSizeInGb=1000
 clientBootDiskSizeInGb=75
 
 publicNetwork=false
 enableGpu=false
-bootstrapLeaderAddress=
+customAddress=
 leaderRotation=true
 
 usage() {
@@ -69,13 +72,16 @@ Manage testnet instances
  create-specific options:
    -n [number]      - Number of additional fullnodes (default: $additionalFullNodeCount)
    -c [number]      - Number of client nodes (default: $clientNodeCount)
+   -u               - Include an API node (default: $apiNode)
    -P               - Use public network IP addresses (default: $publicNetwork)
    -g               - Enable GPU (default: $enableGpu)
-   -G               - Enable GPU, and set count/type of GPUs to use (e.g $cpuBootstrapLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80)
+   -G               - Enable GPU, and set count/type of GPUs to use
-   -a [address]     - Set the bootstreap fullnode's external IP address to this value.
+                      (e.g $cpuBootstrapLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80)
-                      For GCE, [address] is the "name" of the desired External
+   -a [address]     - Address to be be assigned to the API node if present,
+                      otherwise the bootstrap fullnode.
+                      * For GCE, [address] is the "name" of the desired External
                         IP Address.
-                      For EC2, [address] is the "allocation ID" of the desired
+                      * For EC2, [address] is the "allocation ID" of the desired
                         Elastic IP.
    -d [disk-type]   - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
    -b               - Disable leader rotation
@@ -100,7 +106,7 @@ shift
 [[ $command = create || $command = config || $command = info || $command = delete ]] ||
   usage "Invalid command: $command"
 
-while getopts "h?p:Pn:c:z:gG:a:d:b" opt; do
+while getopts "h?p:Pn:c:z:gG:a:d:bu" opt; do
   case $opt in
   h | \?)
     usage
@@ -133,11 +139,14 @@ while getopts "h?p:Pn:c:z:gG:a:d:b" opt; do
     bootstrapLeaderMachineType="$OPTARG"
     ;;
   a)
-    bootstrapLeaderAddress=$OPTARG
+    customAddress=$OPTARG
     ;;
   d)
     bootDiskType=$OPTARG
     ;;
+  u)
+    apiNode=true
+    ;;
   *)
     usage "unhandled option: $opt"
     ;;
@@ -327,6 +336,15 @@ EOF
     cloud_ForEachInstance waitForStartupComplete
   }
 
+  echo "apiIpList=()" >> "$configFile"
+  echo "apiIpListPrivate=()" >> "$configFile"
+  echo "Looking for api instances..."
+  cloud_FindInstances "$prefix-api"
+  [[ ${#instances[@]} -eq 0 ]] || {
+    cloud_ForEachInstance recordInstanceIp apiIpList
+    cloud_ForEachInstance waitForStartupComplete
+  }
+
   echo "Wrote $configFile"
   $metricsWriteDatapoint "testnet-deploy net-config-complete=1"
 }
@@ -382,6 +400,7 @@ Network composition:
   Bootstrap leader = $bootstrapLeaderMachineType (GPU=$enableGpu)
   Additional fullnodes = $additionalFullNodeCount x $fullNodeMachineType
   Client(s) = $clientNodeCount x $clientMachineType
+  API Node = $apiNode
 
 Leader rotation: $leaderRotation
 
@@ -451,6 +470,12 @@ touch /.instance-startup-complete
 
 EOF
 
+  if $apiNode; then
+    apiNodeAddress=$customAddress
+  else
+    bootstrapLeaderAddress=$customAddress
+  fi
+
   cloud_CreateInstances "$prefix" "$prefix-bootstrap-leader" 1 \
     "$imageName" "$bootstrapLeaderMachineType" "$fullNodeBootDiskSizeInGb" \
     "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType"
@@ -465,6 +490,12 @@ EOF
       "$startupScript" "" "$bootDiskType"
   fi
 
+  if $apiNode; then
+    cloud_CreateInstances "$prefix" "$prefix-api" "1" \
+      "$imageName" "$apiNodeMachineType" "$fullNodeBootDiskSizeInGb" \
+      "$startupScript" "$apiNodeAddress" "$bootDiskType"
+  fi
+
   $metricsWriteDatapoint "testnet-deploy net-create-complete=1"
 
   prepareInstancesAndWriteConfigFile
@@ -497,6 +528,12 @@ info)
     ipAddressPrivate=${clientIpListPrivate[$i]}
     printNode bench-tps "$ipAddress" "$ipAddressPrivate"
   done
+
+  for i in $(seq 0 $(( ${#apiIpList[@]} - 1)) ); do
+    ipAddress=${apiIpList[$i]}
+    ipAddressPrivate=${apiIpListPrivate[$i]}
+    printNode api "$ipAddress" "$ipAddressPrivate"
+  done
   ;;
 *)
   usage "Unknown command: $command"

net/net.sh

@@ -225,9 +225,10 @@ startBootstrapLeader() {
 
 startNode() {
   declare ipAddress=$1
+  declare nodeType=$2
   declare logFile="$netLogDir/fullnode-$ipAddress.log"
 
-  echo "--- Starting fullnode: $ipAddress"
+  echo "--- Starting $nodeType: $ipAddress"
   echo "start log: $logFile"
   (
     set -x
@@ -235,7 +236,7 @@ startNode() {
     ssh "${sshOptions[@]}" -n "$ipAddress" \
       "./solana/net/remote/remote-node.sh \
          $deployMethod \
-         fullnode \
+         $nodeType \
          $publicNetwork \
          $entrypointIp \
          ${#fullnodeIpList[@]} \
@@ -347,8 +348,13 @@ start() {
     $metricsWriteDatapoint "testnet-deploy net-start-begin=1"
   fi
 
-  bootstrapLeader=true
+  declare bootstrapLeader=true
-  for ipAddress in "${fullnodeIpList[@]}"; do
+  declare nodeType=fullnode
+  for ipAddress in "${fullnodeIpList[@]}" - "${apiIpList[@]}"; do
+    if [[ $ipAddress = - ]]; then
+      nodeType=apinode
+      continue
+    fi
     if $updateNodes; then
       stopNode "$ipAddress"
     fi
@@ -364,7 +370,7 @@ start() {
       pids=()
       loopCount=0
     else
-      startNode "$ipAddress"
+      startNode "$ipAddress" $nodeType
 
       # Stagger additional node start time. If too many nodes start simultaneously
       # the bootstrap node gets more rsync requests from the additional nodes than
@@ -431,7 +437,7 @@ start() {
   echo
   echo "+++ Deployment Successful"
   echo "Bootstrap leader deployment took $bootstrapNodeDeployTime seconds"
-  echo "Additional fullnode deployment (${#fullnodeIpList[@]} instances) took $additionalNodeDeployTime seconds"
+  echo "Additional fullnode deployment (${#fullnodeIpList[@]} full nodes, ${#apiIpList[@]} api nodes) took $additionalNodeDeployTime seconds"
   echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds"
   echo "Network start logs in $netLogDir:"
   ls -l "$netLogDir"
@@ -466,7 +472,7 @@ stop() {
   SECONDS=0
   $metricsWriteDatapoint "testnet-deploy net-stop-begin=1"
 
-  for ipAddress in "${fullnodeIpList[@]}" "${clientIpList[@]}"; do
+  for ipAddress in "${fullnodeIpList[@]}" "${apiIpList[@]}" "${clientIpList[@]}"; do
     stopNode "$ipAddress"
   done
 
@@ -515,6 +521,9 @@ logs)
   for ipAddress in "${clientIpList[@]}"; do
     fetchRemoteLog "$ipAddress" client
   done
+  for ipAddress in "${apiIpList[@]}"; do
+    fetchRemoteLog "$ipAddress" fullnode
+  done
   ;;
 
 *)

net/remote/remote-node.sh

@@ -98,14 +98,21 @@ snap)
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
   fi
 
-  if [[ $nodeType = bootstrap-leader ]]; then
+  case $nodeType in
+  bootstrap-leader)
     nodeConfig="mode=bootstrap-leader+drone $commonNodeConfig"
     ln -sf -T /var/snap/solana/current/bootstrap-leader/current fullnode.log
     ln -sf -T /var/snap/solana/current/drone/current drone.log
-  else
+    ;;
+  fullnode)
     nodeConfig="mode=fullnode $commonNodeConfig"
     ln -sf -T /var/snap/solana/current/fullnode/current fullnode.log
-  fi
+    ;;
+  *)
+    echo "Error: unknown node type: $nodeType"
+    exit 1
+    ;;
+  esac
 
   logmarker="solana deploy $(date)/$RANDOM"
   logger "$logmarker"
@@ -163,7 +170,7 @@ local|tar)
     ./multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation > bootstrap-leader.log 2>&1 &
     ln -sTf bootstrap-leader.log fullnode.log
     ;;
-  fullnode)
+  fullnode|apinode)
     net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/
 
     if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-fullnode-cuda ]]; then
@@ -171,11 +178,25 @@ local|tar)
       export SOLANA_CUDA=1
     fi
 
+    args=("$maybeNoLeaderRotation")
+    if [[ $nodeType = apinode ]]; then
+      args+=(--entry-stream /tmp/solana-entry-stream.sock)
+    fi
+
     set -x
     if [[ $skipSetup != true ]]; then
       ./multinode-demo/setup.sh -t fullnode $setupArgs
+
+      if [[ $nodeType = apinode ]]; then
+        npm install @solana/blockexplorer
       fi
-    ./multinode-demo/fullnode.sh $maybeNoLeaderRotation "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 &
+    fi
+
+    # Run blockexplorer as root so it can bind to port 80
+    # shellcheck disable=SC2024 # "sudo doesn't affect redirects" warning does not apply
+    sudo npx solana-blockexplorer > blockexplorer.log 2>&1 &
+
+    ./multinode-demo/fullnode.sh "${args[@]}" "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 &
     ;;
   *)
     echo "Error: unknown node type: $nodeType"

net/ssh.sh

@@ -63,6 +63,15 @@ else
   done
 fi
 echo
+echo API nodes:
+if [[ ${#apiIpList[@]} -eq 0 ]]; then
+  echo "  None"
+else
+  for ipAddress in "${apiIpList[@]}"; do
+    printNode fullnode "$ipAddress"
+  done
+fi
+echo
 echo "Use |scp.sh| to transfer files to and from nodes"
 echo