From 1759968c1e5c44cf8a22941a8359b416add10005 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Thu, 17 Oct 2019 15:44:45 -0600 Subject: [PATCH] Colo: Put NVMe disks to use (#6357) automerge --- ci/localnet-sanity.sh | 6 ++---- multinode-demo/clear-config.sh | 19 ------------------- multinode-demo/common.sh | 18 ++---------------- net/common.sh | 28 +++++++++++++++++++++++++++- net/remote/remote-node.sh | 5 +++-- net/scripts/colo-node-onfree-sh | 4 ++++ net/scripts/colo-utils.sh | 4 ++++ 7 files changed, 42 insertions(+), 42 deletions(-) delete mode 100755 multinode-demo/clear-config.sh diff --git a/ci/localnet-sanity.sh b/ci/localnet-sanity.sh index ae27cf3fa0..42dcbe1e55 100755 --- a/ci/localnet-sanity.sh +++ b/ci/localnet-sanity.sh @@ -70,6 +70,7 @@ done source ci/upload-ci-artifact.sh source scripts/configure-metrics.sh +source multinode-demo/common.sh nodes=( "multinode-demo/drone.sh" @@ -173,7 +174,6 @@ startNodes() { waitForNodeToInit "$initCompleteFile" ( - source multinode-demo/common.sh set -x $solana_cli --keypair config/bootstrap-leader/identity-keypair.json \ --url http://127.0.0.1:8899 get-genesis-blockhash @@ -277,7 +277,6 @@ verifyLedger() { for ledger in bootstrap-leader validator; do echo "--- $ledger ledger verification" ( - source multinode-demo/common.sh set -x $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/$ledger verify ) || flag_error @@ -312,7 +311,7 @@ flag_error() { } if ! $skipSetup; then - multinode-demo/clear-config.sh + clear_config_dir "$SOLANA_CONFIG_DIR" multinode-demo/setup.sh else verifyLedger @@ -322,7 +321,6 @@ lastTransactionCount= while [[ $iteration -le $iterations ]]; do echo "--- Node count ($iteration)" ( - source multinode-demo/common.sh set -x client_keypair=/tmp/client-id.json-$$ $solana_keygen new -f -o $client_keypair || exit $? diff --git a/multinode-demo/clear-config.sh b/multinode-demo/clear-config.sh deleted file mode 100755 index 79a6532e87..0000000000 --- a/multinode-demo/clear-config.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# -# Clear the current cluster configuration -# - -here=$(dirname "$0") -# shellcheck source=multinode-demo/common.sh -source "$here"/common.sh - -set -e - -( - set -x - rm -rf "${SOLANA_CONFIG_DIR:?}/" # <-- $i might be a symlink, rm the other side of it first - rm -rf "$SOLANA_CONFIG_DIR" - mkdir -p "$SOLANA_CONFIG_DIR" -) - -setup_secondary_mount diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index c89c306efd..48dcb66992 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -7,7 +7,8 @@ # shellcheck disable=2034 # -SOLANA_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. || exit 1; pwd)" +# shellcheck source=net/common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. || exit 1; pwd)"/net/common.sh if [[ $(uname) != Linux ]]; then # Protect against unsupported configurations to prevent non-obvious errors @@ -62,21 +63,6 @@ solana_replicator=$(solana_program replicator) export RUST_BACKTRACE=1 -# shellcheck source=scripts/configure-metrics.sh -source "$SOLANA_ROOT"/scripts/configure-metrics.sh - -SOLANA_CONFIG_DIR=$SOLANA_ROOT/config - -SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk -setup_secondary_mount() { - # If there is a secondary disk, symlink the config/ dir there - if [[ -d $SECONDARY_DISK_MOUNT_POINT ]]; then - mkdir -p $SECONDARY_DISK_MOUNT_POINT/config - rm -rf "$SOLANA_CONFIG_DIR" - ln -sfT $SECONDARY_DISK_MOUNT_POINT/config "$SOLANA_CONFIG_DIR" - fi -} - default_arg() { declare name=$1 declare value=$2 diff --git a/net/common.sh b/net/common.sh index 73d6257085..12501aa174 100644 --- a/net/common.sh +++ b/net/common.sh @@ -15,8 +15,9 @@ netConfigDir="$netDir"/config netLogDir="$netDir"/log mkdir -p "$netConfigDir" "$netLogDir" +SOLANA_ROOT="$netDir"/.. # shellcheck source=scripts/configure-metrics.sh -source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh +source "$SOLANA_ROOT"/scripts/configure-metrics.sh configFile="$netConfigDir/config" geoipConfigFile="$netConfigDir/geoip.yml" @@ -94,3 +95,28 @@ urlencode() { done } +SOLANA_CONFIG_DIR=$SOLANA_ROOT/config +# Clear the current cluster configuration +clear_config_dir() { + declare config_dir="$1" + ( + set -x + rm -rf "${config_dir:?}/" # <-- $i might be a symlink, rm the other side of it first + rm -rf "$config_dir" + mkdir -p "$config_dir" + ) + + setup_secondary_mount +} + +SECONDARY_DISK_MOUNT_POINT=/mnt/extra-disk +setup_secondary_mount() { + # If there is a secondary disk, symlink the config/ dir there + if [[ -d $SECONDARY_DISK_MOUNT_POINT ]] && \ + [[ -w $SECONDARY_DISK_MOUNT_POINT ]]; then + mkdir -p $SECONDARY_DISK_MOUNT_POINT/config + rm -rf "$SOLANA_CONFIG_DIR" + ln -sfT $SECONDARY_DISK_MOUNT_POINT/config "$SOLANA_CONFIG_DIR" + fi +} + diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 8ca48e9ecd..6c02a6fdbd 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -61,6 +61,7 @@ airdropsEnabled=$airdropsEnabled EOF source net/common.sh +source multinode-demo/common.sh loadConfigFile initCompleteFile=init-complete-node.log @@ -149,7 +150,7 @@ EOF bootstrap-leader) set -x if [[ $skipSetup != true ]]; then - multinode-demo/clear-config.sh + clear_config_dir "$SOLANA_CONFIG_DIR" if [[ -n $internalNodesLamports ]]; then echo "---" >> config/fullnode-balances.yml @@ -245,7 +246,7 @@ EOF net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/version.yml ~/version.yml fi if [[ $skipSetup != true ]]; then - multinode-demo/clear-config.sh + clear_config_dir "$SOLANA_CONFIG_DIR" [[ -z $internalNodesLamports ]] || net/scripts/rsync-retry.sh -vPrc \ "$entrypointIp":~/solana/config/fullnode-"$nodeIndex"-identity.json config/fullnode-identity.json fi diff --git a/net/scripts/colo-node-onfree-sh b/net/scripts/colo-node-onfree-sh index d6348e4883..fcdf8e91fc 100644 --- a/net/scripts/colo-node-onfree-sh +++ b/net/scripts/colo-node-onfree-sh @@ -99,6 +99,10 @@ if [ -f "$SOLANA_LOCK_FILE" ]; then cat > "\${HOME}/.ssh/authorized_keys" < /dev/null) EOAK + EXTERNAL_CONFIG_DIR="${SECONDARY_DISK_MOUNT_POINT}/config/" + if [[ -d "\$EXTERNAL_CONFIG_DIR" ]]; then + rm -rf "\$EXTERNAL_CONFIG_DIR" + fi # End filesystem cleanup RC=true fi diff --git a/net/scripts/colo-utils.sh b/net/scripts/colo-utils.sh index 20e677b5e8..9bd3a9a293 100644 --- a/net/scripts/colo-utils.sh +++ b/net/scripts/colo-utils.sh @@ -3,6 +3,10 @@ declare -r SOLANA_LOCK_FILE="/home/solana/.solana.lock" __colo_here="$(dirname "${BASH_SOURCE[0]}")" + +# shellcheck source=net/common.sh +source "$__colo_here"/../common.sh + # Load colo resource specs export COLO_RES_N=0 export COLO_RES_HOSTNAME=()