solana/net/gce.sh

#!/usr/bin/env bash
set -e

here=$(dirname "$0")
# shellcheck source=net/common.sh
source "$here"/common.sh

cloudProvider=$(basename "$0" .sh)
bootDiskType=""
case $cloudProvider in
gce)
  # shellcheck source=net/scripts/gce-provider.sh
  source "$here"/scripts/gce-provider.sh

  cpuBootstrapLeaderMachineType="--custom-cpu 24 --min-cpu-platform Intel%20Skylake"
  gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=1,type=nvidia-tesla-p100"
  clientMachineType="--custom-cpu 16 --custom-memory 20GB"
  blockstreamerMachineType="--machine-type n1-standard-8"
  selfDestructHours=8
  ;;
ec2)
  # shellcheck source=net/scripts/ec2-provider.sh
  source "$here"/scripts/ec2-provider.sh

  cpuBootstrapLeaderMachineType=m5.4xlarge

  # NOTE: At this time only the p3dn.24xlarge EC2 instance type has GPU and
  #       AVX-512 support.  The default, p2.xlarge, does not support
  #       AVX-512
  gpuBootstrapLeaderMachineType=p2.xlarge
  clientMachineType=c5.2xlarge
  blockstreamerMachineType=m5.4xlarge
  selfDestructHours=0
  ;;
azure)
  # shellcheck source=net/scripts/azure-provider.sh
  source "$here"/scripts/azure-provider.sh

  cpuBootstrapLeaderMachineType=Standard_D16s_v3
  gpuBootstrapLeaderMachineType=Standard_NC12
  clientMachineType=Standard_D16s_v3
  blockstreamerMachineType=Standard_D16s_v3
  selfDestructHours=0
  ;;
colo)
  # shellcheck source=net/scripts/colo-provider.sh
  source "$here"/scripts/colo-provider.sh

  cpuBootstrapLeaderMachineType=0
  gpuBootstrapLeaderMachineType=1
  clientMachineType=0
  blockstreamerMachineType=0
  selfDestructHours=0
  ;;
*)
  echo "Error: Unknown cloud provider: $cloudProvider"
  ;;
esac

prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
additionalValidatorCount=2
clientNodeCount=0
blockstreamer=false
validatorBootDiskSizeInGb=500
clientBootDiskSizeInGb=75
validatorAdditionalDiskSizeInGb=
externalNodes=false
failOnValidatorBootupFailure=true
preemptible=true
evalInfo=false
tmpfsAccounts=false
defaultCustomMemoryGB="$(cloud_DefaultCustomMemoryGB)"
customMemoryGB="$defaultCustomMemoryGB"

publicNetwork=false
letsEncryptDomainName=
enableGpu=false
customMachineType=
customAddress=
zones=()

containsZone() {
  local e match="$1"
  shift
  for e; do [[ "$e" == "$match" ]] && return 0; done
  return 1
}

usage() {
  exitcode=0
  if [[ -n "$1" ]]; then
    exitcode=1
    echo "Error: $*"
  fi
  cat <<EOF
usage: $0 [create|config|delete] [common options] [command-specific options]

Manage testnet instances

 create - create a new testnet (implies 'config')
 config - configure the testnet and write a config file describing it
 delete - delete the testnet
 info   - display information about the currently configured testnet
 status - display status information of all resources

 common options:
   -p [prefix]      - Optional common prefix for instance names to avoid
                      collisions (default: $prefix)
   -z [zone]        - Zone(s) for the nodes (default: $(cloud_DefaultZone))
                      If specified multiple times, the validators will be evenly
                      distributed over all specified zones and
                      client/blockstreamer nodes will be created in the first
                      zone
   -x               - append to the existing configuration instead of creating a
                      new configuration
   --allow-boot-failures
                    - Discard from config validator nodes that didn't bootup
                      successfully

 create-specific options:
   -n [number]      - Number of additional validators (default: $additionalValidatorCount)
   -c [number]      - Number of client nodes (default: $clientNodeCount)
   -u               - Include a Blockstreamer (default: $blockstreamer)
   -P               - Use public network IP addresses (default: $publicNetwork)
   -g               - Enable GPU and automatically set validator machine types to $gpuBootstrapLeaderMachineType
                      (default: $enableGpu)
   -G               - Enable GPU, and set custom GPU machine type to use
                      (e.g $gpuBootstrapLeaderMachineType)
   -a [address]     - Address to be be assigned to the Blockstreamer if present,
                      otherwise the bootstrap validator.
                      * For GCE, [address] is the "name" of the desired External
                        IP Address.
                      * For EC2, [address] is the "allocation ID" of the desired
                        Elastic IP.
   -d [disk-type]   - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
   --letsencrypt [dns name]
                    - Attempt to generate a TLS certificate using this
                      DNS name (useful only when the -a and -P options
                      are also provided)
   --custom-machine-type [type]
                    - Set a custom machine type without assuming whether or not
                      GPU is enabled.  Set this explicitly with --enable-gpu/-g to call out the presence of GPUs.
$(
  if [[ -n "$defaultCustomMemoryGB" ]]; then
    echo "   --custom-memory-gb"
    echo "                    - Set memory size for custom machine type in GB (default: $defaultCustomMemoryGB)"
  fi
)
   --enable-gpu     - Use with --custom-machine-type to specify whether or not GPUs should be used/enabled
   --validator-additional-disk-size-gb [number]
                    - Add an additional [number] GB SSD to all validators to store the config directory.
                      If not set, config will be written to the boot disk by default.
                      Only supported on GCE.
   --dedicated      - Use dedicated instances for additional validators
                      (by default preemptible instances are used to reduce
                      cost).  Note that the bootstrap validator,
                      blockstreamer and client nodes are always dedicated.
                      Set this flag on colo to prevent your testnet from being pre-empted by nightly test automation.
   --self-destruct-hours [number]
                    - Specify lifetime of the allocated instances in hours. 0 to
                      disable. Only supported on GCE. (default: $selfDestructHours)
   --validator-boot-disk-size-gb [number]
                    - Specify validator boot disk size in gb.
   --client-machine-type [type]
                    - custom client machine type
   --tmpfs-accounts - Put accounts directory on a swap-backed tmpfs volume

 config-specific options:
   -P               - Use public network IP addresses (default: $publicNetwork)

 delete-specific options:
   --reclaim-preemptible-reservations
                    - If set, reclaims all reservations on colo nodes that were not created with --dedicated.
                      This behavior does not filter by testnet name or owner.  Only implemented on colo.
   --reclaim-all-reservations
                    - If set, reclaims all reservations on all colo nodes, regardless of owner, pre-emptibility, or creator.

 info-specific options:
   --eval           - Output in a form that can be eval-ed by a shell: eval \$(gce.sh info --eval)

   none

EOF
  exit $exitcode
}

command=$1
[[ -n $command ]] || usage
shift
[[ $command = create || $command = config || $command = info || $command = delete || $command = status ]] ||
  usage "Invalid command: $command"

shortArgs=()
while [[ -n $1 ]]; do
  if [[ ${1:0:2} = -- ]]; then
    if [[ $1 = --letsencrypt ]]; then
      letsEncryptDomainName="$2"
      shift 2
    elif [[ $1 = --validator-additional-disk-size-gb ]]; then
      validatorAdditionalDiskSizeInGb="$2"
      shift 2
    elif [[ $1 == --machine-type* || $1 == --custom-cpu* ]]; then # Bypass quoted long args for GPUs
      shortArgs+=("$1")
      shift
    elif [[ $1 == --allow-boot-failures ]]; then
      failOnValidatorBootupFailure=false
      shift
    elif [[ $1 == --dedicated ]]; then
      preemptible=false
      shift
    elif [[ $1 == --eval ]]; then
      evalInfo=true
      shift
    elif [[ $1 == --enable-gpu ]]; then
      enableGpu=true
      shift
    elif [[ $1 = --custom-machine-type ]]; then
      customMachineType="$2"
      shift 2
    elif [[ $1 = --client-machine-type ]]; then
      clientMachineType="$2"
      shift 2
    elif [[ $1 = --validator-boot-disk-size-gb ]]; then
      validatorBootDiskSizeInGb="$2"
      shift 2
    elif [[ $1 == --self-destruct-hours ]]; then
      maybeTimeout=$2
      if [[ $maybeTimeout =~ ^[0-9]+$ ]]; then
        selfDestructHours=$maybeTimeout
      else
        echo "  Invalid parameter ($maybeTimeout) to $1"
        usage 1
      fi
      shift 2
    elif [[ $1 == --reclaim-preemptible-reservations ]]; then
      reclaimOnlyPreemptibleReservations=true
      shift
    elif [[ $1 == --reclaim-all-reservations ]]; then
      reclaimAllReservations=true
      shift
    elif [[ $1 == --tmpfs-accounts ]]; then
      tmpfsAccounts=true
      shift
    elif [[ $1 == --custom-memory-gb ]]; then
      customMemoryGB=$2
      shift 2
    else
      usage "Unknown long option: $1"
    fi
  else
    shortArgs+=("$1")
    shift
  fi
done

while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do
  case $opt in
  h | \?)
    usage
    ;;
  p)
    [[ ${OPTARG//[^A-Za-z0-9-]/} == "$OPTARG" ]] || usage "Invalid prefix: \"$OPTARG\", alphanumeric only"
    prefix=$OPTARG
    ;;
  P)
    publicNetwork=true
    ;;
  n)
    additionalValidatorCount=$OPTARG
    ;;
  c)
    clientNodeCount=$OPTARG
    ;;
  z)
    containsZone "$OPTARG" "${zones[@]}" || zones+=("$OPTARG")
    ;;
  g)
    enableGpu=true
    ;;
  G)
    enableGpu=true
    customMachineType="$OPTARG"
    ;;
  a)
    customAddress=$OPTARG
    ;;
  d)
    bootDiskType=$OPTARG
    ;;
  u)
    blockstreamer=true
    ;;
  x)
    externalNodes=true
    ;;
  *)
    usage "unhandled option: $opt"
    ;;
  esac
done

[[ ${#zones[@]} -gt 0 ]] || zones+=("$(cloud_DefaultZone)")

[[ -z $1 ]] || usage "Unexpected argument: $1"
if [[ $cloudProvider = ec2 ]]; then
  # EC2 keys can't be retrieved from running instances like GCE keys can so save
  # EC2 keys in the user's home directory so |./ec2.sh config| can at least be
  # used on the same host that ran |./ec2.sh create| .
  sshPrivateKey="$HOME/.ssh/solana-net-id_$prefix"
else
  sshPrivateKey="$netConfigDir/id_$prefix"
fi

case $cloudProvider in
gce)
  if [[ "$tmpfsAccounts" = "true" ]]; then
    cpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
    gpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
    if [[ $customMemoryGB -lt 100 ]]; then
      # shellcheck disable=SC2016 # We don't want expression expansion on these backticks
      echo -e '\nWarning: At least 100GB of system RAM is recommending with `--tmpfs-accounts` (see `--custom-memory-gb`)\n'
    fi
  fi
  cpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
  gpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
  ;;
ec2|azure|colo)
  if [[ -n $validatorAdditionalDiskSizeInGb ]] ; then
    usage "--validator-additional-disk-size-gb currently only supported with cloud provider: gce"
  fi
  if [[ "$tmpfsAccounts" = "true" ]]; then
    usage "--tmpfs-accounts only supported on cloud provider: gce"
  fi
  if [[ "$customMemoryGB" != "$defaultCustomMemoryGB" ]]; then
    usage "--custom-memory-gb only supported on cloud provider: gce"
  fi
  ;;
*)
  echo "Error: Unknown cloud provider: $cloudProvider"
  ;;
esac

case $cloudProvider in
  gce | ec2 | azure)
    maybePreemptible="never preemptible"
    ;;
  colo)
    maybePreemptible=$preemptible
    ;;
  *)
    echo "Error: Unknown cloud provider: $cloudProvider"
    ;;
esac

if [[ $reclaimOnlyPreemptibleReservations == "true" && $reclaimAllReservations == "true" ]]; then
  usage "Cannot set both --reclaim-preemptible-reservations and --reclaim-all-reservations.  Set one or none"
fi

if [[ -n $reclaimAllReservations || -n $reclaimOnlyPreemptibleReservations ]]; then
  forceDelete="true"
fi

if [[ -n "$customMachineType" ]] ; then
  bootstrapLeaderMachineType="$customMachineType"
elif [[ "$enableGpu" = "true" ]] ; then
  bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType"
else
  bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType"
fi
validatorMachineType=$bootstrapLeaderMachineType
blockstreamerMachineType=$bootstrapLeaderMachineType

# cloud_ForEachInstance [cmd] [extra args to cmd]
#
# Execute a command for each element in the `instances` array
#
#   cmd   - The command to execute on each instance
#           The command will receive arguments followed by any
#           additional arguments supplied to cloud_ForEachInstance:
#               name     - name of the instance
#               publicIp - The public IP address of this instance
#               privateIp - The private IP address of this instance
#               zone     - Zone of this instance
#               count    - Monotonically increasing count for each
#                          invocation of cmd, starting at 1
#               ...      - Extra args to cmd..
#
#
cloud_ForEachInstance() {
  declare cmd="$1"
  shift
  [[ -n $cmd ]] || { echo cloud_ForEachInstance: cmd not specified; exit 1; }

  declare count=1
  for info in "${instances[@]}"; do
    declare name publicIp privateIp
    IFS=: read -r name publicIp privateIp zone < <(echo "$info")

    eval "$cmd" "$name" "$publicIp" "$privateIp" "$zone" "$count" "$@"
    count=$((count + 1))
  done
}

# Given a cloud provider zone, return an approximate lat,long location for the
# data center.  Normal geoip lookups for cloud provider IP addresses are
# sometimes widely inaccurate.
zoneLocation() {
  declare zone="$1"
  case "$zone" in
  us-west1*)
    echo "[45.5946, -121.1787]"
    ;;
  us-central1*)
    echo "[41.2619, -95.8608]"
    ;;
  us-east1*)
    echo "[33.1960, -80.0131]"
    ;;
  asia-east2*)
    echo "[22.3193, 114.1694]"
    ;;
  asia-northeast1*)
    echo "[35.6762, 139.6503]"
    ;;
  asia-northeast2*)
    echo "[34.6937, 135.5023]"
    ;;
  asia-south1*)
    echo "[19.0760, 72.8777]"
    ;;
  asia-southeast1*)
    echo "[1.3404, 103.7090]"
    ;;
  australia-southeast1*)
    echo "[-33.8688, 151.2093]"
    ;;
  europe-north1*)
    echo "[60.5693, 27.1878]"
    ;;
  europe-west2*)
    echo "[51.5074, -0.1278]"
    ;;
  europe-west3*)
    echo "[50.1109, 8.6821]"
    ;;
  europe-west4*)
    echo "[53.4386, 6.8355]"
    ;;
  europe-west6*)
    echo "[47.3769, 8.5417]"
    ;;
  northamerica-northeast1*)
    echo "[45.5017, -73.5673]"
    ;;
  southamerica-east1*)
    echo "[-23.5505, -46.6333]"
    ;;
  *)
    ;;
  esac
}

prepareInstancesAndWriteConfigFile() {
  $metricsWriteDatapoint "testnet-deploy net-config-begin=1"

  if $externalNodes; then
    echo "Appending to existing config file"
    echo "externalNodeSshKey=$sshPrivateKey" >> "$configFile"
  else
    rm -f "$geoipConfigFile"
    cat >> "$configFile" <<EOF
# autogenerated at $(date)
netBasename=$prefix
publicNetwork=$publicNetwork
sshPrivateKey=$sshPrivateKey
letsEncryptDomainName=$letsEncryptDomainName
export TMPFS_ACCOUNTS=$tmpfsAccounts
EOF
  fi
  touch "$geoipConfigFile"

  buildSshOptions

  cloud_RestartPreemptedInstances "$prefix"

  fetchPrivateKey() {
    declare nodeName
    declare nodeIp
    declare nodeZone
    IFS=: read -r nodeName nodeIp _ nodeZone < <(echo "${instances[0]}")

    # Make sure the machine is alive or pingable
    timeout_sec=90
    cloud_WaitForInstanceReady "$nodeName" "$nodeIp" "$nodeZone" "$timeout_sec"

    if [[ ! -r $sshPrivateKey ]]; then
      echo "Fetching $sshPrivateKey from $nodeName"

      # Try to scp in a couple times, sshd may not yet be up even though the
      # machine can be pinged...
      (
        set -o pipefail
        for i in $(seq 1 60); do
          set -x
          cloud_FetchFile "$nodeName" "$nodeIp" /solana-scratch/id_ecdsa "$sshPrivateKey" "$nodeZone" &&
            cloud_FetchFile "$nodeName" "$nodeIp" /solana-scratch/id_ecdsa.pub "$sshPrivateKey.pub" "$nodeZone" &&
              break
          set +x

          sleep 1
          echo "Retry $i..."
        done
      )

      chmod 400 "$sshPrivateKey"
      ls -l "$sshPrivateKey"
    fi
  }

  recordInstanceIp() {
    declare name="$1"
    declare publicIp="$2"
    declare privateIp="$3"
    declare zone="$4"
    #declare index="$5"

    declare failOnFailure="$6"
    declare arrayName="$7"

    if [ "$publicIp" = "TERMINATED" ] || [ "$privateIp" = "TERMINATED" ]; then
      if $failOnFailure; then
        exit 1
      else
        return 0
      fi
    fi

    ok=true
    echo "Waiting for $name to finish booting..."
    (
      set +e
      fetchPrivateKey || exit 1
      for i in $(seq 1 60); do
        (
          set -x
          timeout --preserve-status --foreground 20s ssh "${sshOptions[@]}" "$publicIp" "ls -l /solana-scratch/.instance-startup-complete"
        )
        ret=$?
        if [[ $ret -eq 0 ]]; then
          echo "$name has booted."
          exit 0
        fi
        sleep 5
        echo "Retry $i..."
      done
      echo "$name failed to boot."
      exit 1
    ) || ok=false

    if ! $ok; then
      if $failOnFailure; then
        exit 1
      fi
    else
      {
        echo "$arrayName+=($publicIp)  # $name"
        echo "${arrayName}Private+=($privateIp)  # $name"
        echo "${arrayName}Zone+=($zone)  # $name"
      } >> "$configFile"

      declare latlng=
      latlng=$(zoneLocation "$zone")
      if [[ -n $latlng ]]; then
        echo "$publicIp: $latlng" >> "$geoipConfigFile"
      fi
    fi
  }

  if $externalNodes; then
    echo "Bootstrap validator is already configured"
  else
    echo "Looking for bootstrap validator instance..."
    cloud_FindInstance "$prefix-bootstrap-validator"
    [[ ${#instances[@]} -eq 1 ]] || {
      echo "Unable to find bootstrap validator"
      exit 1
    }

    echo "validatorIpList=()" >> "$configFile"
    echo "validatorIpListPrivate=()" >> "$configFile"
    cloud_ForEachInstance recordInstanceIp true validatorIpList
  fi

  if [[ $additionalValidatorCount -gt 0 ]]; then
    numZones=${#zones[@]}
    if [[ $additionalValidatorCount -gt $numZones ]]; then
      numNodesPerZone=$((additionalValidatorCount / numZones))
      numLeftOverNodes=$((additionalValidatorCount % numZones))
    else
      numNodesPerZone=1
      numLeftOverNodes=0
    fi

    for ((i=((numZones - 1)); i >= 0; i--)); do
      zone=${zones[i]}
      if [[ $i -eq 0 ]]; then
        numNodesPerZone=$((numNodesPerZone + numLeftOverNodes))
      fi
      echo "Looking for additional validator instances in $zone ..."
      cloud_FindInstances "$prefix-$zone-validator"
      declare numInstances=${#instances[@]}
      if [[ $numInstances -ge $numNodesPerZone || ( ! $failOnValidatorBootupFailure && $numInstances -gt 0 ) ]]; then
        cloud_ForEachInstance recordInstanceIp "$failOnValidatorBootupFailure" validatorIpList
      else
        echo "Unable to find additional validators"
        if $failOnValidatorBootupFailure; then
          exit 1
        fi
      fi
    done
  fi

  if ! $externalNodes; then
    echo "clientIpList=()" >> "$configFile"
    echo "clientIpListPrivate=()" >> "$configFile"
  fi
  echo "Looking for client bencher instances..."
  cloud_FindInstances "$prefix-client"
  [[ ${#instances[@]} -eq 0 ]] || {
    cloud_ForEachInstance recordInstanceIp true clientIpList
  }

  if ! $externalNodes; then
    echo "blockstreamerIpList=()" >> "$configFile"
    echo "blockstreamerIpListPrivate=()" >> "$configFile"
  fi
  echo "Looking for blockstreamer instances..."
  cloud_FindInstances "$prefix-blockstreamer"
  [[ ${#instances[@]} -eq 0 ]] || {
    cloud_ForEachInstance recordInstanceIp true blockstreamerIpList
  }

  echo "Wrote $configFile"
  $metricsWriteDatapoint "testnet-deploy net-config-complete=1"
}

delete() {
  $metricsWriteDatapoint "testnet-deploy net-delete-begin=1"

  case $cloudProvider in
    gce | ec2 | azure)
      # Filter for all nodes
      filter="$prefix-"
      ;;
    colo)
      if [[ -n $forceDelete ]]; then
        filter=".*-"
      else
        filter="$prefix-"
      fi
      ;;
    *)
      echo "Error: Unknown cloud provider: $cloudProvider"
      ;;
  esac

  echo "Searching for instances: $filter"
  cloud_FindInstances "$filter" "$reclaimOnlyPreemptibleReservations"

  if [[ ${#instances[@]} -eq 0 ]]; then
    echo "No instances found matching '$filter'"
  else
    cloud_DeleteInstances $forceDelete
  fi

  wait

  if $externalNodes; then
    echo "Let's not delete the current configuration file"
  else
    rm -f "$configFile"
  fi

  $metricsWriteDatapoint "testnet-deploy net-delete-complete=1"
}

create_error_cleanup() {
  declare RC=$?
  if [[ "$RC" -ne 0 ]]; then
    delete
  fi
  exit $RC
}

case $command in
delete)
  delete
  ;;

create)
  [[ -n $additionalValidatorCount ]] || usage "Need number of nodes"

  delete

  $metricsWriteDatapoint "testnet-deploy net-create-begin=1"

  if $failOnValidatorBootupFailure; then
    trap create_error_cleanup EXIT
  fi

  rm -rf "$sshPrivateKey"{,.pub}

  # Note: using rsa because |aws ec2 import-key-pair| seems to fail for ecdsa
  ssh-keygen -t rsa -N '' -f "$sshPrivateKey"

  printNetworkInfo() {
    cat <<EOF
==[ Network composition ]===============================================================
  Bootstrap validator = $bootstrapLeaderMachineType (GPU=$enableGpu)
  Additional validators = $additionalValidatorCount x $validatorMachineType
  Client(s) = $clientNodeCount x $clientMachineType
  Blockstreamer = $blockstreamer
========================================================================================

EOF
  }
  printNetworkInfo

  creationDate=$(date)
  creationInfo() {
    cat <<EOF

  Instance running since: $creationDate

========================================================================================
EOF
  }

  declare startupScript="$netConfigDir"/instance-startup-script.sh
  cat > "$startupScript" <<EOF
#!/usr/bin/env bash
# autogenerated at $(date)
set -ex

if [[ -f /solana-scratch/.instance-startup-complete ]]; then
  echo reboot
  $(
    cd "$here"/scripts/
    if "$enableGpu"; then
      cat enable-nvidia-persistence-mode.sh
    fi

    if [[ -n $validatorAdditionalDiskSizeInGb ]]; then
      cat mount-additional-disk.sh
    fi

    cat ../../scripts/ulimit-n.sh
  )
  if [[ -x ~solana/solana/on-reboot ]]; then
    sudo -u solana ~solana/solana/on-reboot
  fi

  # Skip most setup on instance reboot
  exit 0
fi

cat > /etc/motd <<EOM
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

  This instance has not been fully configured.

  See startup script log messages in /var/log/syslog for status:
    $ sudo cat /var/log/syslog | egrep \\(startup-script\\|cloud-init\)

  To block until setup is complete, run:
    $ until [[ -f /solana-scratch/.instance-startup-complete ]]; do sleep 1; done

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
$(creationInfo)
EOM

# Place the generated private key at /solana-scratch/id_ecdsa so it's retrievable by anybody
# who is able to log into this machine
mkdir -m 0777 /solana-scratch
cat > /solana-scratch/id_ecdsa <<EOK
$(cat "$sshPrivateKey")
EOK
cat > /solana-scratch/id_ecdsa.pub <<EOK
$(cat "$sshPrivateKey.pub")
EOK
chmod 444 /solana-scratch/id_ecdsa

USER=\$(id -un)
export DEBIAN_FRONTEND=noninteractive
$(
  cd "$here"/scripts/
  cat \
    disable-background-upgrades.sh \
    create-solana-user.sh \
    solana-user-authorized_keys.sh \
    add-testnet-solana-user-authorized_keys.sh \
    install-ag.sh \
    install-certbot.sh \
    install-earlyoom.sh \
    install-iftop.sh \
    install-libssl-compatability.sh \
    install-nodejs.sh \
    install-redis.sh \
    install-rsync.sh \
    localtime.sh \
    network-config.sh \
    remove-docker-interface.sh \

  if "$enableGpu"; then
    cat enable-nvidia-persistence-mode.sh
  fi

  if [[ -n $validatorAdditionalDiskSizeInGb ]]; then
    cat mount-additional-disk.sh
  fi

  if [[ $selfDestructHours -gt 0 ]]; then
    cat <<EOSD

# Setup GCE self-destruct
cat >/solana-scratch/gce-self-destruct.sh <<'EOS'
$(cat gce-self-destruct.sh)
EOS
EOSD
    cat <<'EOSD'

# Populate terminal prompt update script
cat >/solana-scratch/gce-self-destruct-ps1.sh <<'EOS'
#!/usr/bin/env bash
source "$(dirname "$0")/gce-self-destruct.sh"
gce_self_destruct_ps1
EOS
chmod +x /solana-scratch/gce-self-destruct-ps1.sh

# Append MOTD and PS1 replacement to .profile
cat >>~solana/.profile <<'EOS'

# Print self-destruct countdown on login
source "/solana-scratch/gce-self-destruct.sh"
gce_self_destruct_motd

# Add self-destruct countdown to terminal prompt
export PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]$(/solana-scratch/gce-self-destruct-ps1.sh):\[\033[01;34m\]\w\[\033[00m\]\$ '
EOS
EOSD
    cat <<EOSD

source /solana-scratch/gce-self-destruct.sh
gce_self_destruct_setup $selfDestructHours
EOSD
  fi
)

cat > /etc/motd <<EOM
See startup script log messages in /var/log/syslog for status:
  $ sudo cat /var/log/syslog | egrep \\(startup-script\\|cloud-init\)
$(printNetworkInfo)
$(creationInfo)
EOM

$(
  if [[ "$tmpfsAccounts" = "true" ]]; then
    cat <<'EOSWAP'

# Setup swap/tmpfs for accounts
tmpfsMountPoint=/mnt/solana-accounts
swapDevice="/dev/nvme0n1"
swapUUID="43076c54-7840-4e59-a368-2d164f8984fb"
mkswap --uuid "$swapUUID" "$swapDevice"
echo "UUID=$swapUUID swap swap defaults 0 0" >> /etc/fstab
swapon "UUID=$swapUUID"
mkdir -p -m 0777 "$tmpfsMountPoint"
echo "tmpfs $tmpfsMountPoint tmpfs defaults,size=300G 0 0" >> /etc/fstab
mount "$tmpfsMountPoint"
EOSWAP
  fi
)

touch /solana-scratch/.instance-startup-complete

EOF

  if $blockstreamer; then
    blockstreamerAddress=$customAddress
  else
    bootstrapLeaderAddress=$customAddress
  fi

  for zone in "${zones[@]}"; do
    cloud_Initialize "$prefix" "$zone"
  done

  if $externalNodes; then
    echo "Bootstrap validator is already configured"
  else
    cloud_CreateInstances "$prefix" "$prefix-bootstrap-validator" 1 \
      "$enableGpu" "$bootstrapLeaderMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \
      "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" "$validatorAdditionalDiskSizeInGb" \
      "$maybePreemptible" "$sshPrivateKey"
  fi

  if [[ $additionalValidatorCount -gt 0 ]]; then
    num_zones=${#zones[@]}
    if [[ $additionalValidatorCount -gt $num_zones ]]; then
      numNodesPerZone=$((additionalValidatorCount / num_zones))
      numLeftOverNodes=$((additionalValidatorCount % num_zones))
    else
      numNodesPerZone=1
      numLeftOverNodes=0
    fi

    for ((i=((num_zones - 1)); i >= 0; i--)); do
      zone=${zones[i]}
      if [[ $i -eq 0 ]]; then
        numNodesPerZone=$((numNodesPerZone + numLeftOverNodes))
      fi
      cloud_CreateInstances "$prefix" "$prefix-$zone-validator" "$numNodesPerZone" \
        "$enableGpu" "$validatorMachineType" "$zone" "$validatorBootDiskSizeInGb" \
        "$startupScript" "" "$bootDiskType" "$validatorAdditionalDiskSizeInGb" \
        "$preemptible" "$sshPrivateKey" &
    done

    wait
  fi

  if [[ $clientNodeCount -gt 0 ]]; then
    cloud_CreateInstances "$prefix" "$prefix-client" "$clientNodeCount" \
      "$enableGpu" "$clientMachineType" "${zones[0]}" "$clientBootDiskSizeInGb" \
      "$startupScript" "" "$bootDiskType" "" "$maybePreemptible" "$sshPrivateKey"
  fi

  if $blockstreamer; then
    cloud_CreateInstances "$prefix" "$prefix-blockstreamer" "1" \
      "$enableGpu" "$blockstreamerMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \
      "$startupScript" "$blockstreamerAddress" "$bootDiskType" "" "$maybePreemptible" "$sshPrivateKey"
  fi

  $metricsWriteDatapoint "testnet-deploy net-create-complete=1"

  prepareInstancesAndWriteConfigFile
  ;;

config)
  failOnValidatorBootupFailure=false
  prepareInstancesAndWriteConfigFile
  ;;
info)
  loadConfigFile
  printNode() {
    declare nodeType=$1
    declare ip=$2
    declare ipPrivate=$3
    declare zone=$4
    printf "  %-16s | %-15s | %-15s | %s\n" "$nodeType" "$ip" "$ipPrivate" "$zone"
  }

  if $evalInfo; then
    echo "NET_NUM_VALIDATORS=${#validatorIpList[@]}"
    echo "NET_NUM_CLIENTS=${#clientIpList[@]}"
    echo "NET_NUM_BLOCKSTREAMERS=${#blockstreamerIpList[@]}"
  else
    printNode "Node Type" "Public IP" "Private IP" "Zone"
    echo "-------------------+-----------------+-----------------+--------------"
  fi

  nodeType=bootstrap-validator
  if [[ ${#validatorIpList[@]} -gt 0 ]]; then
    for i in $(seq 0 $(( ${#validatorIpList[@]} - 1)) ); do
      ipAddress=${validatorIpList[$i]}
      ipAddressPrivate=${validatorIpListPrivate[$i]}
      zone=${validatorIpListZone[$i]}
      if $evalInfo; then
        echo "NET_VALIDATOR${i}_IP=$ipAddress"
      else
        printNode $nodeType "$ipAddress" "$ipAddressPrivate" "$zone"
      fi
      nodeType=validator
    done
  fi

  if [[ ${#clientIpList[@]} -gt 0 ]]; then
    for i in $(seq 0 $(( ${#clientIpList[@]} - 1)) ); do
      ipAddress=${clientIpList[$i]}
      ipAddressPrivate=${clientIpListPrivate[$i]}
      zone=${clientIpListZone[$i]}
      if $evalInfo; then
        echo "NET_CLIENT${i}_IP=$ipAddress"
      else
        printNode client "$ipAddress" "$ipAddressPrivate" "$zone"
      fi
    done
  fi

  if [[ ${#blockstreamerIpList[@]} -gt 0 ]]; then
    for i in $(seq 0 $(( ${#blockstreamerIpList[@]} - 1)) ); do
      ipAddress=${blockstreamerIpList[$i]}
      ipAddressPrivate=${blockstreamerIpListPrivate[$i]}
      zone=${blockstreamerIpListZone[$i]}
      if $evalInfo; then
        echo "NET_BLOCKSTREAMER${i}_IP=$ipAddress"
      else
        printNode blockstreamer "$ipAddress" "$ipAddressPrivate" "$zone"
      fi
    done
  fi

  ;;
status)
  cloud_StatusAll
  ;;
*)
  usage "Unknown command: $command"
esac