Support account on tmpfs via net/ scripts (bp #14459) (#14620)

* multinode-demo: Pass --accounts through bootstrap leader wrapper

(cherry picked from commit 327be55acc)

* gce.sh: Factor out default custom memory

(cherry picked from commit ddf1d2dbf5)

* net/: Support accounts on swap-backed tmpfs

(cherry picked from commit ff599ace4d)

* net/gce.sh: Add cusom RAM arg instead of doubling default with tmpfs

(cherry picked from commit 3175cf1deb)

Co-authored-by: Trent Nelson <trent@solana.com>

multinode-demo/bootstrap-validator.sh

@@ -66,6 +66,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --expected-bank-hash ]]; then
       args+=("$1" "$2")
       shift 2
+    elif [[ $1 == --accounts ]]; then
+      args+=("$1" "$2")
+      shift 2
     else
       echo "Unknown argument: $1"
       $program --help

net/gce.sh

@@ -12,7 +12,7 @@ gce)
   # shellcheck source=net/scripts/gce-provider.sh
   source "$here"/scripts/gce-provider.sh
 
-  cpuBootstrapLeaderMachineType="--custom-cpu 24 --custom-memory 64GB --min-cpu-platform Intel%20Skylake"
+  cpuBootstrapLeaderMachineType="--custom-cpu 24 --min-cpu-platform Intel%20Skylake"
   gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=1,type=nvidia-tesla-p100"
   clientMachineType="--custom-cpu 16 --custom-memory 20GB"
   blockstreamerMachineType="--machine-type n1-standard-8"
@@ -68,6 +68,9 @@ externalNodes=false
 failOnValidatorBootupFailure=true
 preemptible=true
 evalInfo=false
+tmpfsAccounts=false
+defaultCustomMemoryGB="$(cloud_DefaultCustomMemoryGB)"
+customMemoryGB="$defaultCustomMemoryGB"
 
 publicNetwork=false
 letsEncryptDomainName=
@@ -137,6 +140,12 @@ Manage testnet instances
    --custom-machine-type
                     - Set a custom machine type without assuming whether or not
                       GPU is enabled.  Set this explicitly with --enable-gpu/-g to call out the presence of GPUs.
+$(
+  if [[ -n "$defaultCustomMemoryGB" ]]; then
+    echo "   --custom-memory-gb"
+    echo "                    - Set memory size for custom machine type in GB (default: $defaultCustomMemoryGB)"
+  fi
+)
    --enable-gpu     - Use with --custom-machine-type to specify whether or not GPUs should be used/enabled
    --validator-additional-disk-size-gb [number]
                     - Add an additional [number] GB SSD to all validators to store the config directory.
@@ -150,6 +159,7 @@ Manage testnet instances
    --self-destruct-hours [number]
                     - Specify lifetime of the allocated instances in hours. 0 to
                       disable. Only supported on GCE. (default: $selfDestructHours)
+   --tmpfs-accounts - Put accounts directory on a swap-backed tmpfs volume
 
  config-specific options:
    -P               - Use public network IP addresses (default: $publicNetwork)
@@ -218,6 +228,12 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --reclaim-all-reservations ]]; then
       reclaimAllReservations=true
       shift
+    elif [[ $1 == --tmpfs-accounts ]]; then
+      tmpfsAccounts=true
+      shift
+    elif [[ $1 == --custom-memory-gb ]]; then
+      customMemoryGB=$2
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -273,16 +289,6 @@ while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do
   esac
 done
 
-if [[ -n "$customMachineType" ]] ; then
-  bootstrapLeaderMachineType="$customMachineType"
-elif [[ "$enableGpu" = "true" ]] ; then
-  bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType"
-else
-  bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType"
-fi
-validatorMachineType=$bootstrapLeaderMachineType
-blockstreamerMachineType=$bootstrapLeaderMachineType
-
 [[ ${#zones[@]} -gt 0 ]] || zones+=("$(cloud_DefaultZone)")
 
 [[ -z $1 ]] || usage "Unexpected argument: $1"
@@ -297,10 +303,26 @@ fi
 
 case $cloudProvider in
 gce)
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    cpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
+    gpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
+    if [[ $customMemoryGB -lt 100 ]]; then
+      # shellcheck disable=SC2016 # We don't want expression expansion on these backticks
+      echo -e '\nWarning: At least 100GB of system RAM is recommending with `--tmpfs-accounts` (see `--custom-memory-gb`)\n'
+    fi
+  fi
+  cpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
+  gpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
   ;;
 ec2|azure|colo)
   if [[ -n $validatorAdditionalDiskSizeInGb ]] ; then
-    usage "Error: --validator-additional-disk-size-gb currently only supported with cloud provider: gce"
+    usage "--validator-additional-disk-size-gb currently only supported with cloud provider: gce"
+  fi
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    usage "--tmpfs-accounts only supported on cloud provider: gce"
+  fi
+  if [[ "$customMemoryGB" != "$defaultCustomMemoryGB" ]]; then
+    usage "--custom-memory-gb only supported on cloud provider: gce"
   fi
   ;;
 *)
@@ -328,6 +350,16 @@ if [[ -n $reclaimAllReservations || -n $reclaimOnlyPreemptibleReservations ]]; t
   forceDelete="true"
 fi
 
+if [[ -n "$customMachineType" ]] ; then
+  bootstrapLeaderMachineType="$customMachineType"
+elif [[ "$enableGpu" = "true" ]] ; then
+  bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType"
+else
+  bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType"
+fi
+validatorMachineType=$bootstrapLeaderMachineType
+blockstreamerMachineType=$bootstrapLeaderMachineType
+
 # cloud_ForEachInstance [cmd] [extra args to cmd]
 #
 # Execute a command for each element in the `instances` array
@@ -432,6 +464,7 @@ netBasename=$prefix
 publicNetwork=$publicNetwork
 sshPrivateKey=$sshPrivateKey
 letsEncryptDomainName=$letsEncryptDomainName
+export TMPFS_ACCOUNTS=$tmpfsAccounts
 EOF
   fi
   touch "$geoipConfigFile"
@@ -820,6 +853,24 @@ $(printNetworkInfo)
 $(creationInfo)
 EOM
 
+$(
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    cat <<'EOSWAP'
+
+# Setup swap/tmpfs for accounts
+tmpfsMountPoint=/mnt/solana-accounts
+swapDevice="/dev/nvme0n1"
+swapUUID="43076c54-7840-4e59-a368-2d164f8984fb"
+mkswap --uuid "$swapUUID" "$swapDevice"
+echo "UUID=$swapUUID swap swap defaults 0 0" >> /etc/fstab
+swapon "UUID=$swapUUID"
+mkdir -p -m 0777 "$tmpfsMountPoint"
+echo "tmpfs $tmpfsMountPoint tmpfs defaults,size=300G 0 0" >> /etc/fstab
+mount "$tmpfsMountPoint"
+EOSWAP
+  fi
+)
+
 touch /solana-scratch/.instance-startup-complete
 
 EOF

net/net.sh

@@ -289,6 +289,7 @@ startBootstrapLeader() {
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
          \"$extraPrimordialStakes\" \
+         \"$TMPFS_ACCOUNTS\" \
       "
 
   ) >> "$logFile" 2>&1 || {
@@ -360,6 +361,7 @@ startNode() {
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
          \"$extraPrimordialStakes\" \
+         \"$TMPFS_ACCOUNTS\" \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!

net/remote/remote-node.sh

@@ -28,6 +28,7 @@ gpuMode="${19:-auto}"
 maybeWarpSlot="${20}"
 waitForNodeInit="${21}"
 extraPrimordialStakes="${22:=0}"
+tmpfsAccounts="${23:false}"
 set +x
 
 missing() {
@@ -274,6 +275,10 @@ EOF
       --init-complete-file "$initCompleteFile"
     )
 
+    if [[ "$tmpfsAccounts" = "true" ]]; then
+      args+=(--accounts /mnt/solana-accounts)
+    fi
+
     if [[ $airdropsEnabled = true ]]; then
 cat >> ~/solana/on-reboot <<EOF
       ./multinode-demo/faucet.sh > faucet.log 2>&1 &
@@ -391,6 +396,10 @@ EOF
       maybeSkipAccountsCreation="export SKIP_ACCOUNTS_CREATION=1"
     fi
 
+    if [[ "$tmpfsAccounts" = "true" ]]; then
+      args+=(--accounts /mnt/solana-accounts)
+    fi
+
 cat >> ~/solana/on-reboot <<EOF
     $maybeSkipAccountsCreation
     nohup multinode-demo/validator.sh ${args[@]} > validator.log.\$now 2>&1 &

net/scripts/azure-provider.sh

@@ -8,6 +8,10 @@ cloud_DefaultZone() {
   echo "westus"
 }
 
+cloud_DefaultCustomMemoryGB() {
+  : # Not implemented
+}
+
 cloud_RestartPreemptedInstances() {
   : # Not implemented
 }

net/scripts/colo-provider.sh

@@ -17,6 +17,10 @@ cloud_DefaultZone() {
   echo "Denver"
 }
 
+cloud_DefaultCustomMemoryGB() {
+  : # Not implemented
+}
+
 cloud_RestartPreemptedInstances() {
   : # Not implemented
 }

net/scripts/ec2-provider.sh

@@ -7,6 +7,10 @@ cloud_DefaultZone() {
   echo "us-east-1b"
 }
 
+cloud_DefaultCustomMemoryGB() {
+  : # Not implemented
+}
+
 cloud_RestartPreemptedInstances() {
   : # Not implemented
 }

net/scripts/gce-provider.sh

@@ -8,6 +8,10 @@ cloud_DefaultZone() {
   echo "us-west1-b"
 }
 
+cloud_DefaultCustomMemoryGB() {
+  echo 64
+}
+
 #
 # cloud_RestartPreemptedInstances [namePrefix]
 #