diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 87ce93eae7..d938211890 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -66,6 +66,9 @@ while [[ -n $1 ]]; do elif [[ $1 == --expected-bank-hash ]]; then args+=("$1" "$2") shift 2 + elif [[ $1 == --accounts ]]; then + args+=("$1" "$2") + shift 2 else echo "Unknown argument: $1" $program --help diff --git a/net/gce.sh b/net/gce.sh index 5114641921..18bc103e09 100755 --- a/net/gce.sh +++ b/net/gce.sh @@ -12,7 +12,7 @@ gce) # shellcheck source=net/scripts/gce-provider.sh source "$here"/scripts/gce-provider.sh - cpuBootstrapLeaderMachineType="--custom-cpu 24 --custom-memory 64GB --min-cpu-platform Intel%20Skylake" + cpuBootstrapLeaderMachineType="--custom-cpu 24 --min-cpu-platform Intel%20Skylake" gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=1,type=nvidia-tesla-p100" clientMachineType="--custom-cpu 16 --custom-memory 20GB" blockstreamerMachineType="--machine-type n1-standard-8" @@ -68,6 +68,9 @@ externalNodes=false failOnValidatorBootupFailure=true preemptible=true evalInfo=false +tmpfsAccounts=false +defaultCustomMemoryGB="$(cloud_DefaultCustomMemoryGB)" +customMemoryGB="$defaultCustomMemoryGB" publicNetwork=false letsEncryptDomainName= @@ -137,6 +140,12 @@ Manage testnet instances --custom-machine-type [type] - Set a custom machine type without assuming whether or not GPU is enabled. Set this explicitly with --enable-gpu/-g to call out the presence of GPUs. +$( + if [[ -n "$defaultCustomMemoryGB" ]]; then + echo " --custom-memory-gb" + echo " - Set memory size for custom machine type in GB (default: $defaultCustomMemoryGB)" + fi +) --enable-gpu - Use with --custom-machine-type to specify whether or not GPUs should be used/enabled --validator-additional-disk-size-gb [number] - Add an additional [number] GB SSD to all validators to store the config directory. @@ -154,6 +163,7 @@ Manage testnet instances - Specify validator boot disk size in gb. --client-machine-type [type] - custom client machine type + --tmpfs-accounts - Put accounts directory on a swap-backed tmpfs volume config-specific options: -P - Use public network IP addresses (default: $publicNetwork) @@ -228,6 +238,12 @@ while [[ -n $1 ]]; do elif [[ $1 == --reclaim-all-reservations ]]; then reclaimAllReservations=true shift + elif [[ $1 == --tmpfs-accounts ]]; then + tmpfsAccounts=true + shift + elif [[ $1 == --custom-memory-gb ]]; then + customMemoryGB=$2 + shift 2 else usage "Unknown long option: $1" fi @@ -283,16 +299,6 @@ while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do esac done -if [[ -n "$customMachineType" ]] ; then - bootstrapLeaderMachineType="$customMachineType" -elif [[ "$enableGpu" = "true" ]] ; then - bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType" -else - bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType" -fi -validatorMachineType=$bootstrapLeaderMachineType -blockstreamerMachineType=$bootstrapLeaderMachineType - [[ ${#zones[@]} -gt 0 ]] || zones+=("$(cloud_DefaultZone)") [[ -z $1 ]] || usage "Unexpected argument: $1" @@ -307,10 +313,26 @@ fi case $cloudProvider in gce) + if [[ "$tmpfsAccounts" = "true" ]]; then + cpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme" + gpuBootstrapLeaderMachineType+=" --local-ssd interface=nvme" + if [[ $customMemoryGB -lt 100 ]]; then + # shellcheck disable=SC2016 # We don't want expression expansion on these backticks + echo -e '\nWarning: At least 100GB of system RAM is recommending with `--tmpfs-accounts` (see `--custom-memory-gb`)\n' + fi + fi + cpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB" + gpuBootstrapLeaderMachineType+=" --custom-memory ${customMemoryGB}GB" ;; ec2|azure|colo) if [[ -n $validatorAdditionalDiskSizeInGb ]] ; then - usage "Error: --validator-additional-disk-size-gb currently only supported with cloud provider: gce" + usage "--validator-additional-disk-size-gb currently only supported with cloud provider: gce" + fi + if [[ "$tmpfsAccounts" = "true" ]]; then + usage "--tmpfs-accounts only supported on cloud provider: gce" + fi + if [[ "$customMemoryGB" != "$defaultCustomMemoryGB" ]]; then + usage "--custom-memory-gb only supported on cloud provider: gce" fi ;; *) @@ -338,6 +360,16 @@ if [[ -n $reclaimAllReservations || -n $reclaimOnlyPreemptibleReservations ]]; t forceDelete="true" fi +if [[ -n "$customMachineType" ]] ; then + bootstrapLeaderMachineType="$customMachineType" +elif [[ "$enableGpu" = "true" ]] ; then + bootstrapLeaderMachineType="$gpuBootstrapLeaderMachineType" +else + bootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType" +fi +validatorMachineType=$bootstrapLeaderMachineType +blockstreamerMachineType=$bootstrapLeaderMachineType + # cloud_ForEachInstance [cmd] [extra args to cmd] # # Execute a command for each element in the `instances` array @@ -442,6 +474,7 @@ netBasename=$prefix publicNetwork=$publicNetwork sshPrivateKey=$sshPrivateKey letsEncryptDomainName=$letsEncryptDomainName +export TMPFS_ACCOUNTS=$tmpfsAccounts EOF fi touch "$geoipConfigFile" @@ -830,6 +863,24 @@ $(printNetworkInfo) $(creationInfo) EOM +$( + if [[ "$tmpfsAccounts" = "true" ]]; then + cat <<'EOSWAP' + +# Setup swap/tmpfs for accounts +tmpfsMountPoint=/mnt/solana-accounts +swapDevice="/dev/nvme0n1" +swapUUID="43076c54-7840-4e59-a368-2d164f8984fb" +mkswap --uuid "$swapUUID" "$swapDevice" +echo "UUID=$swapUUID swap swap defaults 0 0" >> /etc/fstab +swapon "UUID=$swapUUID" +mkdir -p -m 0777 "$tmpfsMountPoint" +echo "tmpfs $tmpfsMountPoint tmpfs defaults,size=300G 0 0" >> /etc/fstab +mount "$tmpfsMountPoint" +EOSWAP + fi +) + touch /solana-scratch/.instance-startup-complete EOF diff --git a/net/net.sh b/net/net.sh index a6b9755d6f..c054555168 100755 --- a/net/net.sh +++ b/net/net.sh @@ -294,6 +294,7 @@ startBootstrapLeader() { \"$maybeWarpSlot\" \ \"$waitForNodeInit\" \ \"$extraPrimordialStakes\" \ + \"$TMPFS_ACCOUNTS\" \ " ) >> "$logFile" 2>&1 || { @@ -365,6 +366,7 @@ startNode() { \"$maybeWarpSlot\" \ \"$waitForNodeInit\" \ \"$extraPrimordialStakes\" \ + \"$TMPFS_ACCOUNTS\" \ " ) >> "$logFile" 2>&1 & declare pid=$! diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index b3de1a792b..c468d7202e 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -28,6 +28,7 @@ gpuMode="${19:-auto}" maybeWarpSlot="${20}" waitForNodeInit="${21}" extraPrimordialStakes="${22:=0}" +tmpfsAccounts="${23:false}" set +x missing() { @@ -275,6 +276,10 @@ EOF --init-complete-file "$initCompleteFile" ) + if [[ "$tmpfsAccounts" = "true" ]]; then + args+=(--accounts /mnt/solana-accounts) + fi + if [[ $airdropsEnabled = true ]]; then cat >> ~/solana/on-reboot < faucet.log 2>&1 & @@ -392,6 +397,10 @@ EOF maybeSkipAccountsCreation="export SKIP_ACCOUNTS_CREATION=1" fi + if [[ "$tmpfsAccounts" = "true" ]]; then + args+=(--accounts /mnt/solana-accounts) + fi + cat >> ~/solana/on-reboot < validator.log.\$now 2>&1 & diff --git a/net/scripts/azure-provider.sh b/net/scripts/azure-provider.sh index b6597c120e..e4d2c2bad2 100755 --- a/net/scripts/azure-provider.sh +++ b/net/scripts/azure-provider.sh @@ -8,6 +8,10 @@ cloud_DefaultZone() { echo "westus" } +cloud_DefaultCustomMemoryGB() { + : # Not implemented +} + cloud_RestartPreemptedInstances() { : # Not implemented } diff --git a/net/scripts/colo-provider.sh b/net/scripts/colo-provider.sh index b16a6754fd..c0a58821c8 100755 --- a/net/scripts/colo-provider.sh +++ b/net/scripts/colo-provider.sh @@ -17,6 +17,10 @@ cloud_DefaultZone() { echo "Denver" } +cloud_DefaultCustomMemoryGB() { + : # Not implemented +} + cloud_RestartPreemptedInstances() { : # Not implemented } diff --git a/net/scripts/ec2-provider.sh b/net/scripts/ec2-provider.sh index f7acf33499..365b8119b9 100755 --- a/net/scripts/ec2-provider.sh +++ b/net/scripts/ec2-provider.sh @@ -7,6 +7,10 @@ cloud_DefaultZone() { echo "us-east-1b" } +cloud_DefaultCustomMemoryGB() { + : # Not implemented +} + cloud_RestartPreemptedInstances() { : # Not implemented } diff --git a/net/scripts/gce-provider.sh b/net/scripts/gce-provider.sh index 9f76cf020b..dbedcea864 100755 --- a/net/scripts/gce-provider.sh +++ b/net/scripts/gce-provider.sh @@ -8,6 +8,10 @@ cloud_DefaultZone() { echo "us-west1-b" } +cloud_DefaultCustomMemoryGB() { + echo 64 +} + # # cloud_RestartPreemptedInstances [namePrefix] #