Support account on tmpfs via net/ scripts (bp #14459) (#14621)

* multinode-demo: Pass --accounts through bootstrap leader wrapper

(cherry picked from commit 327be55acc)

* gce.sh: Factor out default custom memory

(cherry picked from commit ddf1d2dbf5)

* net/: Support accounts on swap-backed tmpfs

(cherry picked from commit ff599ace4d)

* net/gce.sh: Add cusom RAM arg instead of doubling default with tmpfs

(cherry picked from commit 3175cf1deb)

Co-authored-by: Trent Nelson <trent@solana.com>

net/gce.sh

@@ -12,7 +12,7 @@ gce)
   # shellcheck source=net/scripts/gce-provider.sh
   source "$here"/scripts/gce-provider.sh
 
-  cpuBootstrapLeaderMachineType="--custom-cpu 24 --custom-memory 64GB --min-cpu-platform Intel%20Skylake"
+  cpuBootstrapLeaderMachineType="--custom-cpu 24 --min-cpu-platform Intel%20Skylake"
   gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=1,type=nvidia-tesla-p100"
   clientMachineType="--custom-cpu 16 --custom-memory 20GB"
   blockstreamerMachineType="--machine-type n1-standard-8"
@@ -68,6 +68,9 @@ externalNodes=false
 failOnValidatorBootupFailure=true
 preemptible=true
 evalInfo=false
+tmpfsAccounts=false
+defaultCustomMemoryGB="$(cloud_DefaultCustomMemoryGB)"
+customMemoryGB="$defaultCustomMemoryGB"
 
 publicNetwork=false
 letsEncryptDomainName=
@@ -137,6 +140,12 @@ Manage testnet instances
    --custom-machine-type [type]
                     - Set a custom machine type without assuming whether or not
                       GPU is enabled.  Set this explicitly with --enable-gpu/-g to call out the presence of GPUs.
+$(
+  if [[ -n "$defaultCustomMemoryGB" ]]; then
+    echo "   --custom-memory-gb"
+    echo "                    - Set memory size for custom machine type in GB (default: $defaultCustomMemoryGB)"
+  fi
+)
    --enable-gpu     - Use with --custom-machine-type to specify whether or not GPUs should be used/enabled
    --validator-additional-disk-size-gb [number]
                     - Add an additional [number] GB SSD to all validators to store the config directory.
@@ -154,6 +163,7 @@ Manage testnet instances
                     - Specify validator boot disk size in gb.
    --client-machine-type [type]
                     - custom client machine type
+   --tmpfs-accounts - Put accounts directory on a swap-backed tmpfs volume
 
  config-specific options:
    -P               - Use public network IP addresses (default: $publicNetwork)
@@ -228,6 +238,12 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --reclaim-all-reservations ]]; then
       reclaimAllReservations=true
       shift
+    elif [[ $1 == --tmpfs-accounts ]]; then
+      tmpfsAccounts=true
+      shift
+    elif [[ $1 == --custom-memory-gb ]]; then
+      customMemoryGB=$2
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -283,16 +299,6 @@ while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do
   esac
 done
 
-if [[ -n "$customMachineType" ]] ; then
-  bootstrapLeaderMachineType="$customMachineType"
-elif [[ "$enableGpu" = "true" ]] ; then
-  bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType"
-else
-  bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType"
-fi
-validatorMachineType=$bootstrapLeaderMachineType
-blockstreamerMachineType=$bootstrapLeaderMachineType
-
 [[ ${#zones[@]} -gt 0 ]] || zones+=("$(cloud_DefaultZone)")
 
 [[ -z $1 ]] || usage "Unexpected argument: $1"
@@ -307,10 +313,26 @@ fi
 
 case $cloudProvider in
 gce)
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    cpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
+    gpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme"
+    if [[ $customMemoryGB -lt 100 ]]; then
+      # shellcheck disable=SC2016 # We don't want expression expansion on these backticks
+      echo -e '\nWarning: At least 100GB of system RAM is recommending with `--tmpfs-accounts` (see `--custom-memory-gb`)\n'
+    fi
+  fi
+  cpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
+  gpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB"
   ;;
 ec2|azure|colo)
   if [[ -n $validatorAdditionalDiskSizeInGb ]] ; then
-    usage "Error: --validator-additional-disk-size-gb currently only supported with cloud provider: gce"
+    usage "--validator-additional-disk-size-gb currently only supported with cloud provider: gce"
+  fi
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    usage "--tmpfs-accounts only supported on cloud provider: gce"
+  fi
+  if [[ "$customMemoryGB" != "$defaultCustomMemoryGB" ]]; then
+    usage "--custom-memory-gb only supported on cloud provider: gce"
   fi
   ;;
 *)
@@ -338,6 +360,16 @@ if [[ -n $reclaimAllReservations || -n $reclaimOnlyPreemptibleReservations ]]; t
   forceDelete="true"
 fi
 
+if [[ -n "$customMachineType" ]] ; then
+  bootstrapLeaderMachineType="$customMachineType"
+elif [[ "$enableGpu" = "true" ]] ; then
+  bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType"
+else
+  bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType"
+fi
+validatorMachineType=$bootstrapLeaderMachineType
+blockstreamerMachineType=$bootstrapLeaderMachineType
+
 # cloud_ForEachInstance [cmd] [extra args to cmd]
 #
 # Execute a command for each element in the `instances` array
@@ -442,6 +474,7 @@ netBasename=$prefix
 publicNetwork=$publicNetwork
 sshPrivateKey=$sshPrivateKey
 letsEncryptDomainName=$letsEncryptDomainName
+export TMPFS_ACCOUNTS=$tmpfsAccounts
 EOF
   fi
   touch "$geoipConfigFile"
@@ -830,6 +863,24 @@ $(printNetworkInfo)
 $(creationInfo)
 EOM
 
+$(
+  if [[ "$tmpfsAccounts" = "true" ]]; then
+    cat <<'EOSWAP'
+
+# Setup swap/tmpfs for accounts
+tmpfsMountPoint=/mnt/solana-accounts
+swapDevice="/dev/nvme0n1"
+swapUUID="43076c54-7840-4e59-a368-2d164f8984fb"
+mkswap --uuid "$swapUUID" "$swapDevice"
+echo "UUID=$swapUUID swap swap defaults 0 0" >> /etc/fstab
+swapon "UUID=$swapUUID"
+mkdir -p -m 0777 "$tmpfsMountPoint"
+echo "tmpfs $tmpfsMountPoint tmpfs defaults,size=300G 0 0" >> /etc/fstab
+mount "$tmpfsMountPoint"
+EOSWAP
+  fi
+)
+
 touch /solana-scratch/.instance-startup-complete
 
 EOF