Colo: Put NVMe disks to use (#6357)

automerge

net/common.sh

@@ -15,8 +15,9 @@ netConfigDir="$netDir"/config
 netLogDir="$netDir"/log
 mkdir -p "$netConfigDir" "$netLogDir"
 
+SOLANA_ROOT="$netDir"/..
 # shellcheck source=scripts/configure-metrics.sh
-source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
+source "$SOLANA_ROOT"/scripts/configure-metrics.sh
 
 configFile="$netConfigDir/config"
 geoipConfigFile="$netConfigDir/geoip.yml"
@@ -94,3 +95,28 @@ urlencode() {
   done
 }
 
+SOLANA_CONFIG_DIR=$SOLANA_ROOT/config
+# Clear the current cluster configuration
+clear_config_dir() {
+  declare config_dir="$1"
+  (
+    set -x
+    rm -rf "${config_dir:?}/" # <-- $i might be a symlink, rm the other side of it first
+    rm -rf "$config_dir"
+    mkdir -p "$config_dir"
+  )
+
+  setup_secondary_mount
+}
+
+SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk
+setup_secondary_mount() {
+  # If there is a secondary disk, symlink the config/ dir there
+  if [[ -d $SECONDARY_DISK_MOUNT_POINT ]] && \
+    [[ -w $SECONDARY_DISK_MOUNT_POINT ]]; then
+    mkdir -p $SECONDARY_DISK_MOUNT_POINT/config
+    rm -rf "$SOLANA_CONFIG_DIR"
+    ln -sfT $SECONDARY_DISK_MOUNT_POINT/config "$SOLANA_CONFIG_DIR"
+  fi
+}
+

net/remote/remote-node.sh

@@ -61,6 +61,7 @@ airdropsEnabled=$airdropsEnabled
 EOF
 
 source net/common.sh
+source multinode-demo/common.sh
 loadConfigFile
 
 initCompleteFile=init-complete-node.log
@@ -149,7 +150,7 @@ EOF
   bootstrap-leader)
     set -x
     if [[ $skipSetup != true ]]; then
-      multinode-demo/clear-config.sh
+      clear_config_dir "$SOLANA_CONFIG_DIR"
 
       if [[ -n $internalNodesLamports ]]; then
         echo "---" >> config/fullnode-balances.yml
@@ -245,7 +246,7 @@ EOF
       net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/version.yml ~/version.yml
     fi
     if [[ $skipSetup != true ]]; then
-      multinode-demo/clear-config.sh
+      clear_config_dir "$SOLANA_CONFIG_DIR"
       [[ -z $internalNodesLamports ]] || net/scripts/rsync-retry.sh -vPrc \
       "$entrypointIp":~/solana/config/fullnode-"$nodeIndex"-identity.json config/fullnode-identity.json
     fi

net/scripts/colo-node-onfree-sh

@@ -99,6 +99,10 @@ if [ -f "$SOLANA_LOCK_FILE" ]; then
     cat > "\${HOME}/.ssh/authorized_keys" <<EOAK
 $("$__colo_here"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)
 EOAK
+    EXTERNAL_CONFIG_DIR="${SECONDARY_DISK_MOUNT_POINT}/config/"
+    if [[ -d "\$EXTERNAL_CONFIG_DIR" ]]; then
+      rm -rf "\$EXTERNAL_CONFIG_DIR"
+    fi
     # End filesystem cleanup
     RC=true
   fi

net/scripts/colo-utils.sh

@@ -3,6 +3,10 @@
 declare -r SOLANA_LOCK_FILE="/home/solana/.solana.lock"
 
 __colo_here="$(dirname "${BASH_SOURCE[0]}")"
+
+# shellcheck source=net/common.sh
+source "$__colo_here"/../common.sh
+
 # Load colo resource specs
 export COLO_RES_N=0
 export COLO_RES_HOSTNAME=()