gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Plumb pre-emptibility and associated overrides into colo allocation and automated testing (#8754)

Browse Source 
automerge
This commit is contained in:
Dan Albert
2020-03-10 11:25:44 -07:00
committed by GitHub
parent 496999beba
commit 595c96b262
6 changed files with 89 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
net/gce.sh
View File

@ -61,7 +61,6 @@ colo)
;; ;;
esac esac
prefix=testnet-dev-${USER//[^A-Za-z0-9]/} prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
additionalValidatorCount=2 additionalValidatorCount=2
clientNodeCount=0 clientNodeCount=0
@ -154,6 +153,7 @@ Manage testnet instances
(by default preemptible instances are used to reduce (by default preemptible instances are used to reduce
cost). Note that the bootstrap validator, archiver, cost). Note that the bootstrap validator, archiver,
blockstreamer and client nodes are always dedicated. blockstreamer and client nodes are always dedicated.
Set this flag on colo to prevent your testnet from being pre-empted by nightly test automation.
--self-destruct-hours [number] --self-destruct-hours [number]
- Specify lifetime of the allocated instances in hours. 0 to - Specify lifetime of the allocated instances in hours. 0 to
disable. Only supported on GCE. (default: $selfDestructHours) disable. Only supported on GCE. (default: $selfDestructHours)
@ -162,7 +162,11 @@ Manage testnet instances
-P - Use public network IP addresses (default: $publicNetwork) -P - Use public network IP addresses (default: $publicNetwork)
delete-specific options: delete-specific options:
none --reclaim-preemptible-reservations
- If set, reclaims all reservations on colo nodes that were not created with --dedicated.
This behavior does not filter by testnet name or owner. Only implemented on colo.
--reclaim-all-reservations
- If set, reclaims all reservations on all colo nodes, regardless of owner, pre-emptibility, or creator.
info-specific options: info-specific options:
--eval - Output in a form that can be eval-ed by a shell: eval $(gce.sh info) --eval - Output in a form that can be eval-ed by a shell: eval $(gce.sh info)
@ -215,6 +219,12 @@ while [[ -n $1 ]]; do
usage 1 usage 1
fi fi
shift 2 shift 2
elif [[ $1 == --reclaim-preemptible-reservations ]]; then
reclaimOnlyPreemptibleReservations=true
shift
elif [[ $1 == --reclaim-all-reservations ]]; then
reclaimAllReservations=true
shift
else else
usage "Unknown long option: $1" usage "Unknown long option: $1"
fi fi
@ -308,6 +318,26 @@ ec2|azure|colo)
;; ;;
esac esac
case $cloudProvider in
gce | ec2 | azure)
maybePreemptible="never preemptible"
;;
colo)
maybePreemptible=$preemptible
;;
*)
echo "Error: Unknown cloud provider: $cloudProvider"
;;
esac
if [[ $reclaimOnlyPreemptibleReservations == "true" && $reclaimAllReservations == "true" ]]; then
usage "Cannot set both --reclaim-preemptible-reservations and --reclaim-all-reservations. Set one or none"
fi
if [[ -n $reclaimAllReservations || -n $reclaimOnlyPreemptibleReservations ]]; then
forceDelete="true"
fi
# cloud_ForEachInstance [cmd] [extra args to cmd] # cloud_ForEachInstance [cmd] [extra args to cmd]
# #
# Execute a command for each element in the `instances` array # Execute a command for each element in the `instances` array
@ -594,16 +624,30 @@ EOF
delete() { delete() {
$metricsWriteDatapoint "testnet-deploy net-delete-begin=1" $metricsWriteDatapoint "testnet-deploy net-delete-begin=1"
# Filter for all nodes case $cloudProvider in
filter="$prefix-" gce | ec2 | azure)
# Filter for all nodes
filter="$prefix-"
;;
colo)
if [[ -n $forceDelete ]]; then
filter=".*-"
else
filter="$prefix-"
fi
;;
*)
echo "Error: Unknown cloud provider: $cloudProvider"
;;
esac
echo "Searching for instances: $filter" echo "Searching for instances: $filter"
cloud_FindInstances "$filter" cloud_FindInstances "$filter" "$reclaimOnlyPreemptibleReservations"
if [[ ${#instances[@]} -eq 0 ]]; then if [[ ${#instances[@]} -eq 0 ]]; then
echo "No instances found matching '$filter'" echo "No instances found matching '$filter'"
else else
cloud_DeleteInstances true & cloud_DeleteInstances $forceDelete
fi fi
wait wait
@ -817,7 +861,7 @@ EOF
cloud_CreateInstances "$prefix" "$prefix-bootstrap-validator" 1 \ cloud_CreateInstances "$prefix" "$prefix-bootstrap-validator" 1 \
"$enableGpu" "$bootstrapLeaderMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \ "$enableGpu" "$bootstrapLeaderMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \
"$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" "$validatorAdditionalDiskSizeInGb" \ "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType" "$validatorAdditionalDiskSizeInGb" \
"never preemptible" "$sshPrivateKey" "$maybePreemptible" "$sshPrivateKey"
fi fi
if [[ $additionalValidatorCount -gt 0 ]]; then if [[ $additionalValidatorCount -gt 0 ]]; then
@ -847,19 +891,19 @@ EOF
if [[ $clientNodeCount -gt 0 ]]; then if [[ $clientNodeCount -gt 0 ]]; then
cloud_CreateInstances "$prefix" "$prefix-client" "$clientNodeCount" \ cloud_CreateInstances "$prefix" "$prefix-client" "$clientNodeCount" \
"$enableGpu" "$clientMachineType" "${zones[0]}" "$clientBootDiskSizeInGb" \ "$enableGpu" "$clientMachineType" "${zones[0]}" "$clientBootDiskSizeInGb" \
"$startupScript" "" "$bootDiskType" "" "never preemptible" "$sshPrivateKey" "$startupScript" "" "$bootDiskType" "" "$maybePreemptible" "$sshPrivateKey"
fi fi
if $blockstreamer; then if $blockstreamer; then
cloud_CreateInstances "$prefix" "$prefix-blockstreamer" "1" \ cloud_CreateInstances "$prefix" "$prefix-blockstreamer" "1" \
"$enableGpu" "$blockstreamerMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \ "$enableGpu" "$blockstreamerMachineType" "${zones[0]}" "$validatorBootDiskSizeInGb" \
"$startupScript" "$blockstreamerAddress" "$bootDiskType" "" "$sshPrivateKey" "$startupScript" "$blockstreamerAddress" "$bootDiskType" "" "$maybePreemptible" "$sshPrivateKey"
fi fi
if [[ $archiverNodeCount -gt 0 ]]; then if [[ $archiverNodeCount -gt 0 ]]; then
cloud_CreateInstances "$prefix" "$prefix-archiver" "$archiverNodeCount" \ cloud_CreateInstances "$prefix" "$prefix-archiver" "$archiverNodeCount" \
false "$archiverMachineType" "${zones[0]}" "$archiverBootDiskSizeInGb" \ false "$archiverMachineType" "${zones[0]}" "$archiverBootDiskSizeInGb" \
"$startupScript" "" "" "" "never preemptible" "$sshPrivateKey" "$startupScript" "" "" "" "$maybePreemptible" "$sshPrivateKey"
fi fi
$metricsWriteDatapoint "testnet-deploy net-create-complete=1" $metricsWriteDatapoint "testnet-deploy net-create-complete=1"

2
net/scripts/colo-node-onacquire.sh
View File

@ -3,6 +3,7 @@
# These variable must be set before the main body is called # These variable must be set before the main body is called
SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}" SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}"
INSTANCE_NAME="${INSTANCE_NAME:?}" INSTANCE_NAME="${INSTANCE_NAME:?}"
PREEMPTIBLE="${PREEMPTIBLE:?}"
SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}" SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}"
SSH_PRIVATE_KEY_TEXT="${SSH_PRIVATE_KEY_TEXT:?}" SSH_PRIVATE_KEY_TEXT="${SSH_PRIVATE_KEY_TEXT:?}"
SSH_PUBLIC_KEY_TEXT="${SSH_PUBLIC_KEY_TEXT:?}" SSH_PUBLIC_KEY_TEXT="${SSH_PUBLIC_KEY_TEXT:?}"
@ -16,6 +17,7 @@ if [[ ! -f "${SOLANA_LOCK_FILE}" ]]; then
{ {
echo "export SOLANA_LOCK_USER=${SOLANA_USER}" echo "export SOLANA_LOCK_USER=${SOLANA_USER}"
echo "export SOLANA_LOCK_INSTANCENAME=${INSTANCE_NAME}" echo "export SOLANA_LOCK_INSTANCENAME=${INSTANCE_NAME}"
echo "export PREEMPTIBLE=${PREEMPTIBLE}"
echo "[[ -v SSH_TTY && -f \"${HOME}/.solana-motd\" ]] && cat \"${HOME}/.solana-motd\" 1>&2" echo "[[ -v SSH_TTY && -f \"${HOME}/.solana-motd\" ]] && cat \"${HOME}/.solana-motd\" 1>&2"
} >&9 } >&9
exec 9>&- exec 9>&-

3
net/scripts/colo-node-onfree.sh
View File

@ -4,6 +4,7 @@
SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}" SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}"
SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT:?}" SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT:?}"
SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}" SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}"
FORCE_DELETE="${FORCE_DELETE}"
RC=false RC=false
if [[ -f "${SOLANA_LOCK_FILE}" ]]; then if [[ -f "${SOLANA_LOCK_FILE}" ]]; then
@ -11,7 +12,7 @@ if [[ -f "${SOLANA_LOCK_FILE}" ]]; then
flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 ) flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 )
# shellcheck disable=SC1090 # shellcheck disable=SC1090
. "${SOLANA_LOCK_FILE}" . "${SOLANA_LOCK_FILE}"
if [[ "${SOLANA_LOCK_USER}" = "${SOLANA_USER}" ]]; then if [[ "${SOLANA_LOCK_USER}" = "${SOLANA_USER}" || -n "${FORCE_DELETE}" ]]; then
# Begin running process cleanup # Begin running process cleanup
CLEANUP_PID=$$ CLEANUP_PID=$$
CLEANUP_PIDS=() CLEANUP_PIDS=()

30
net/scripts/colo-provider.sh
View File

@ -39,6 +39,7 @@ cloud_RestartPreemptedInstances() {
__cloud_FindInstances() { __cloud_FindInstances() {
declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INSTANCES_TEXT declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INSTANCES_TEXT
declare filter=${1} declare filter=${1}
declare onlyPreemptible=${2}
instances=() instances=()
if ! ${COLO_PARALLELIZE}; then if ! ${COLO_PARALLELIZE}; then
@ -47,10 +48,14 @@ __cloud_FindInstances() {
fi fi
INSTANCES_TEXT="$( INSTANCES_TEXT="$(
for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do
IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${AVAIL}" IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE <<<"${AVAIL}"
if [[ ${INSTNAME} =~ ${filter} ]]; then if [[ ${INSTNAME} =~ ${filter} ]]; then
printf "%-40s | publicIp=%-16s privateIp=%s zone=%s\n" "${INSTNAME}" "${IP}" "${PRIV_IP}" "${ZONE}" 1>&2 if [[ -n $onlyPreemptible && $PREEMPTIBLE == "false" ]]; then
echo -e "${INSTNAME}:${IP}:${PRIV_IP}:${ZONE}" continue
else
printf "%-40s | publicIp=%-16s privateIp=%s zone=%s preemptible=%s\n" "${INSTNAME}" "${IP}" "${PRIV_IP}" "${ZONE}" "${PREEMPTIBLE}" 1>&2
echo -e "${INSTNAME}:${IP}:${PRIV_IP}:${ZONE}"
fi
fi fi
done | sort -t $'\v' -k1 done | sort -t $'\v' -k1
)" )"
@ -77,7 +82,8 @@ __cloud_FindInstances() {
# #
cloud_FindInstances() { cloud_FindInstances() {
declare filter="^${1}.*" declare filter="^${1}.*"
__cloud_FindInstances "${filter}" declare onlyPreemptible="${2}"
__cloud_FindInstances "${filter}" "${onlyPreemptible}"
} }
# #
@ -96,7 +102,8 @@ cloud_FindInstances() {
# #
cloud_FindInstance() { cloud_FindInstance() {
declare name="^${1}$" declare name="^${1}$"
__cloud_FindInstances "${name}" declare onlyPreemptible="${2}"
__cloud_FindInstances "${name}" "${onlyPreemptible}"
} }
# #
@ -155,7 +162,7 @@ cloud_CreateInstances() {
#declare optionalAddress="${9}" # unused #declare optionalAddress="${9}" # unused
#declare optionalBootDiskType="${10}" # unused #declare optionalBootDiskType="${10}" # unused
#declare optionalAdditionalDiskSize="${11}" # unused #declare optionalAdditionalDiskSize="${11}" # unused
#declare optionalPreemptible="${12}" # unused declare optionalPreemptible="${12}"
declare sshPrivateKey="${13}" declare sshPrivateKey="${13}"
declare -a nodes declare -a nodes
@ -213,7 +220,7 @@ cloud_CreateInstances() {
RES_MACH="${COLO_RES_MACHINE[${RI}]}" RES_MACH="${COLO_RES_MACHINE[${RI}]}"
IP="${COLO_RES_IP[${RI}]}" IP="${COLO_RES_IP[${RI}]}"
if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then
if colo_node_requisition "${IP}" "${node}" "${sshPrivateKey}" >/dev/null; then if colo_node_requisition "${IP}" "${node}" "${sshPrivateKey}" "${optionalPreemptible}" >/dev/null; then
NI=$((NI+1)) NI=$((NI+1))
fi fi
fi fi
@ -228,10 +235,11 @@ cloud_CreateInstances() {
# Deletes all the instances listed in the `instances` array # Deletes all the instances listed in the `instances` array
# #
cloud_DeleteInstances() { cloud_DeleteInstances() {
declare forceDelete="${1}"
declare _ IP _ _ declare _ IP _ _
for instance in "${instances[@]}"; do for instance in "${instances[@]}"; do
IFS=':' read -r _ IP _ _ <<< "${instance}" IFS=':' read -r _ IP _ _ <<< "${instance}"
colo_node_free "${IP}" >/dev/null colo_node_free "${IP}" "${forceDelete}" >/dev/null
done done
} }
@ -270,13 +278,13 @@ cloud_FetchFile() {
} }
cloud_StatusAll() { cloud_StatusAll() {
declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE
if ! ${COLO_PARALLELIZE}; then if ! ${COLO_PARALLELIZE}; then
colo_load_resources colo_load_resources
colo_load_availability false colo_load_availability false
fi fi
for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do
IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${AVAIL}" IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE <<<"${AVAIL}"
printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s\n" "${HOST_NAME}" "${IP}" "${PRIV_IP}" "${STATUS}" "${LOCK_USER}" "${ZONE}" "${INSTNAME}" printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s preemptible=%s\n" "${HOST_NAME}" "${IP}" "${PRIV_IP}" "${STATUS}" "${LOCK_USER}" "${ZONE}" "${INSTNAME}" "${PREEMPTIBLE}"
done done
} }

18
net/scripts/colo-utils.sh
View File

@ -46,17 +46,17 @@ declare COLO_RES_AVAILABILITY_CACHED=false
declare -ax COLO_RES_AVAILABILITY declare -ax COLO_RES_AVAILABILITY
colo_load_availability() { colo_load_availability() {
declare USE_CACHE=${1:-${COLO_RES_AVAILABILITY_CACHED}} declare USE_CACHE=${1:-${COLO_RES_AVAILABILITY_CACHED}}
declare LINE PRIV_IP STATUS LOCK_USER I IP HOST_NAME ZONE INSTNAME declare LINE PRIV_IP STATUS LOCK_USER I IP HOST_NAME ZONE INSTNAME PREEMPTIBLE
if ! ${USE_CACHE}; then if ! ${USE_CACHE}; then
COLO_RES_AVAILABILITY=() COLO_RES_AVAILABILITY=()
COLO_RES_REQUISITIONED=() COLO_RES_REQUISITIONED=()
while read -r LINE; do while read -r LINE; do
IFS=$'\v' read -r IP STATUS LOCK_USER INSTNAME <<< "${LINE}" IFS=$'\v' read -r IP STATUS LOCK_USER INSTNAME PREEMPTIBLE <<< "${LINE}"
I=$(colo_res_index_from_ip "${IP}") I=$(colo_res_index_from_ip "${IP}")
PRIV_IP="${COLO_RES_IP_PRIV[${I}]}" PRIV_IP="${COLO_RES_IP_PRIV[${I}]}"
HOST_NAME="${COLO_RES_HOSTNAME[${I}]}" HOST_NAME="${COLO_RES_HOSTNAME[${I}]}"
ZONE="${COLO_RES_ZONE[${I}]}" ZONE="${COLO_RES_ZONE[${I}]}"
COLO_RES_AVAILABILITY+=( "$(echo -e "${HOST_NAME}\v${IP}\v${PRIV_IP}\v${STATUS}\v${ZONE}\v${LOCK_USER}\v${INSTNAME}")" ) COLO_RES_AVAILABILITY+=( "$(echo -e "${HOST_NAME}\v${IP}\v${PRIV_IP}\v${STATUS}\v${ZONE}\v${LOCK_USER}\v${INSTNAME}\v${PREEMPTIBLE}")" )
done < <(colo_node_status_all | sort -t $'\v' -k1) done < <(colo_node_status_all | sort -t $'\v' -k1)
COLO_RES_AVAILABILITY_CACHED=true COLO_RES_AVAILABILITY_CACHED=true
fi fi
@ -142,15 +142,15 @@ __colo_node_status_script() {
# the time due to ${SOLANA_LOCK_FILE} not existing and is running from a # the time due to ${SOLANA_LOCK_FILE} not existing and is running from a
# subshell where normal redirection doesn't work # subshell where normal redirection doesn't work
exec 9<"${SOLANA_LOCK_FILE}" && flock -s 9 && . "${SOLANA_LOCK_FILE}" && exec 9>&- exec 9<"${SOLANA_LOCK_FILE}" && flock -s 9 && . "${SOLANA_LOCK_FILE}" && exec 9>&-
echo -e "\${SOLANA_LOCK_USER}\\v\${SOLANA_LOCK_INSTANCENAME}\\vEOL" echo -e "\${SOLANA_LOCK_USER}\\v\${SOLANA_LOCK_INSTANCENAME}\\v\${PREEMPTIBLE}\\vEOL"
exec 2>&3 # Restore stderr exec 2>&3 # Restore stderr
EOF EOF
} }
__colo_node_status_result_normalize() { __colo_node_status_result_normalize() {
declare IP RC US BY INSTNAME EOL declare IP RC US BY INSTNAME PREEMPTIBLE EOL
declare ST="DOWN" declare ST="DOWN"
IFS=$'\v' read -r IP RC US INSTNAME EOL <<< "${1}" IFS=$'\v' read -r IP RC US INSTNAME PREEMPTIBLE EOL <<< "${1}"
if [ "${RC}" -eq 0 ]; then if [ "${RC}" -eq 0 ]; then
[[ "${EOL}" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"${1}\"" 1>&2 [[ "${EOL}" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"${1}\"" 1>&2
if [ -n "${US}" ]; then if [ -n "${US}" ]; then
@ -163,7 +163,7 @@ __colo_node_status_result_normalize() {
ST="FREE" ST="FREE"
fi fi
fi fi
echo -e $"${IP}\v${ST}\v${BY}\v${INSTNAME}" echo -e $"${IP}\v${ST}\v${BY}\v${INSTNAME}\v${PREEMPTIBLE}"
} }
colo_node_status() { colo_node_status() {
@ -188,6 +188,7 @@ colo_node_requisition() {
declare INSTANCE_NAME=${2} declare INSTANCE_NAME=${2}
# shellcheck disable=SC2034 # shellcheck disable=SC2034
declare SSH_PRIVATE_KEY="${3}" declare SSH_PRIVATE_KEY="${3}"
declare PREEMPTIBLE="${4}"
declare INDEX declare INDEX
INDEX=$(colo_res_index_from_ip "${IP}") INDEX=$(colo_res_index_from_ip "${IP}")
@ -196,6 +197,7 @@ colo_node_requisition() {
colo_instance_run "${IP}" "$(cat <<EOF colo_instance_run "${IP}" "$(cat <<EOF
SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE}" SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE}"
INSTANCE_NAME="${INSTANCE_NAME}" INSTANCE_NAME="${INSTANCE_NAME}"
PREEMPTIBLE="${PREEMPTIBLE}"
SSH_AUTHORIZED_KEYS='$("${__colo_here}"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)' SSH_AUTHORIZED_KEYS='$("${__colo_here}"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)'
SSH_PRIVATE_KEY_TEXT="$(<"${SSH_PRIVATE_KEY}")" SSH_PRIVATE_KEY_TEXT="$(<"${SSH_PRIVATE_KEY}")"
SSH_PUBLIC_KEY_TEXT="$(<"${SSH_PRIVATE_KEY}.pub")" SSH_PUBLIC_KEY_TEXT="$(<"${SSH_PRIVATE_KEY}.pub")"
@ -238,10 +240,12 @@ colo_machine_types_compatible() {
colo_node_free() { colo_node_free() {
declare IP=${1} declare IP=${1}
declare FORCE_DELETE=${2}
colo_instance_run "${IP}" "$(cat <<EOF colo_instance_run "${IP}" "$(cat <<EOF
SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE}" SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE}"
SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT}" SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT}"
SSH_AUTHORIZED_KEYS='$("${__colo_here}"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)' SSH_AUTHORIZED_KEYS='$("${__colo_here}"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)'
FORCE_DELETE="${FORCE_DELETE}"
$(<"${__colo_here}"/colo-node-onfree.sh) $(<"${__colo_here}"/colo-node-onfree.sh)
EOF EOF
)" )"

1
system-test/testnet-automation.sh
View File

@ -160,6 +160,7 @@ function launchTestnet() {
${ADDITIONAL_FLAGS[@]/#/" "} ${ADDITIONAL_FLAGS[@]/#/" "}
;; ;;
colo) colo)
net/colo.sh delete --reclaim-preemptible-reservations
# shellcheck disable=SC2068 # shellcheck disable=SC2068
# shellcheck disable=SC2086 # shellcheck disable=SC2086
net/colo.sh create \ net/colo.sh create \