Plumb pre-emptibility and associated overrides into colo allocation and automated testing (#8754)

automerge
2020-03-10 11:25:44 -07:00
parent 496999beba
commit 595c96b262
6 changed files with 89 additions and 29 deletions
--- a/net/scripts/colo-node-onacquire.sh
+++ b/net/scripts/colo-node-onacquire.sh
@ -3,6 +3,7 @@
 # These variable must be set before the main body is called
 SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}"
 INSTANCE_NAME="${INSTANCE_NAME:?}"
+PREEMPTIBLE="${PREEMPTIBLE:?}"
 SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}"
 SSH_PRIVATE_KEY_TEXT="${SSH_PRIVATE_KEY_TEXT:?}"
 SSH_PUBLIC_KEY_TEXT="${SSH_PUBLIC_KEY_TEXT:?}"
@ -16,6 +17,7 @@ if [[ ! -f "${SOLANA_LOCK_FILE}" ]]; then
  {
    echo "export SOLANA_LOCK_USER=${SOLANA_USER}"
    echo "export SOLANA_LOCK_INSTANCENAME=${INSTANCE_NAME}"
+    echo "export PREEMPTIBLE=${PREEMPTIBLE}"
    echo "[[ -v SSH_TTY && -f \"${HOME}/.solana-motd\" ]] && cat \"${HOME}/.solana-motd\" 1>&2"
  } >&9
  exec 9>&-
--- a/net/scripts/colo-node-onfree.sh
+++ b/net/scripts/colo-node-onfree.sh
@ -4,6 +4,7 @@
 SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE:?}"
 SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT:?}"
 SSH_AUTHORIZED_KEYS="${SSH_AUTHORIZED_KEYS:?}"
+FORCE_DELETE="${FORCE_DELETE}"

 RC=false
 if [[ -f "${SOLANA_LOCK_FILE}" ]]; then
@ -11,7 +12,7 @@ if [[ -f "${SOLANA_LOCK_FILE}" ]]; then
  flock -x -n 9 || ( echo "Failed to acquire lock!" 1>&2 && exit 1 )
  # shellcheck disable=SC1090
  . "${SOLANA_LOCK_FILE}"
-  if [[ "${SOLANA_LOCK_USER}" = "${SOLANA_USER}" ]]; then
+  if [[ "${SOLANA_LOCK_USER}" = "${SOLANA_USER}"  || -n "${FORCE_DELETE}" ]]; then
    # Begin running process cleanup
    CLEANUP_PID=$$
    CLEANUP_PIDS=()
--- a/net/scripts/colo-provider.sh
+++ b/net/scripts/colo-provider.sh
@ -39,6 +39,7 @@ cloud_RestartPreemptedInstances() {
 __cloud_FindInstances() {
  declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME INSTANCES_TEXT
  declare filter=${1}
+  declare onlyPreemptible=${2}
  instances=()

  if ! ${COLO_PARALLELIZE}; then
@ -47,10 +48,14 @@ __cloud_FindInstances() {
  fi
  INSTANCES_TEXT="$(
    for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do
-      IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${AVAIL}"
+      IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE <<<"${AVAIL}"
      if [[ ${INSTNAME} =~ ${filter} ]]; then
-        printf "%-40s | publicIp=%-16s privateIp=%s zone=%s\n" "${INSTNAME}" "${IP}" "${PRIV_IP}" "${ZONE}" 1>&2
-        echo -e "${INSTNAME}:${IP}:${PRIV_IP}:${ZONE}"
+        if [[ -n $onlyPreemptible && $PREEMPTIBLE == "false" ]]; then
+          continue
+        else
+          printf "%-40s | publicIp=%-16s privateIp=%s zone=%s preemptible=%s\n" "${INSTNAME}" "${IP}" "${PRIV_IP}" "${ZONE}" "${PREEMPTIBLE}" 1>&2
+          echo -e "${INSTNAME}:${IP}:${PRIV_IP}:${ZONE}"
+        fi
      fi
    done | sort -t $'\v' -k1
  )"
@ -77,7 +82,8 @@ __cloud_FindInstances() {
 #
 cloud_FindInstances() {
  declare filter="^${1}.*"
-  __cloud_FindInstances "${filter}"
+  declare onlyPreemptible="${2}"
+  __cloud_FindInstances "${filter}" "${onlyPreemptible}"
 }

 #
@ -96,7 +102,8 @@ cloud_FindInstances() {
 #
 cloud_FindInstance() {
  declare name="^${1}$"
-  __cloud_FindInstances "${name}"
+  declare onlyPreemptible="${2}"
+  __cloud_FindInstances "${name}" "${onlyPreemptible}"
 }

 #
@ -155,7 +162,7 @@ cloud_CreateInstances() {
  #declare optionalAddress="${9}" # unused
  #declare optionalBootDiskType="${10}" # unused
  #declare optionalAdditionalDiskSize="${11}" # unused
-  #declare optionalPreemptible="${12}" # unused
+  declare optionalPreemptible="${12}"
  declare sshPrivateKey="${13}"

  declare -a nodes
@ -213,7 +220,7 @@ cloud_CreateInstances() {
      RES_MACH="${COLO_RES_MACHINE[${RI}]}"
      IP="${COLO_RES_IP[${RI}]}"
      if colo_machine_types_compatible "${RES_MACH}" "${machineType}"; then
-        if colo_node_requisition "${IP}" "${node}" "${sshPrivateKey}" >/dev/null; then
+        if colo_node_requisition "${IP}" "${node}" "${sshPrivateKey}" "${optionalPreemptible}" >/dev/null; then
          NI=$((NI+1))
        fi
      fi
@ -228,10 +235,11 @@ cloud_CreateInstances() {
 # Deletes all the instances listed in the `instances` array
 #
 cloud_DeleteInstances() {
+  declare forceDelete="${1}"
  declare _ IP _ _
  for instance in "${instances[@]}"; do
    IFS=':' read -r _ IP _ _ <<< "${instance}"
-    colo_node_free "${IP}" >/dev/null
+    colo_node_free "${IP}" "${forceDelete}" >/dev/null
  done
 }

@ -270,13 +278,13 @@ cloud_FetchFile() {
 }

 cloud_StatusAll() {
-  declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME
+  declare HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE
  if ! ${COLO_PARALLELIZE}; then
    colo_load_resources
    colo_load_availability false
  fi
  for AVAIL in "${COLO_RES_AVAILABILITY[@]}"; do
-    IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME <<<"${AVAIL}"
-    printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s\n" "${HOST_NAME}" "${IP}" "${PRIV_IP}" "${STATUS}" "${LOCK_USER}" "${ZONE}" "${INSTNAME}"
+    IFS=$'\v' read -r HOST_NAME IP PRIV_IP STATUS ZONE LOCK_USER INSTNAME PREEMPTIBLE <<<"${AVAIL}"
+    printf "%-30s | publicIp=%-16s privateIp=%s status=%s who=%s zone=%s inst=%s preemptible=%s\n" "${HOST_NAME}" "${IP}" "${PRIV_IP}" "${STATUS}" "${LOCK_USER}" "${ZONE}" "${INSTNAME}" "${PREEMPTIBLE}"
  done
 }
--- a/net/scripts/colo-utils.sh
+++ b/net/scripts/colo-utils.sh
@ -46,17 +46,17 @@ declare COLO_RES_AVAILABILITY_CACHED=false
 declare -ax COLO_RES_AVAILABILITY
 colo_load_availability() {
  declare USE_CACHE=${1:-${COLO_RES_AVAILABILITY_CACHED}}
-  declare LINE PRIV_IP STATUS LOCK_USER I IP HOST_NAME ZONE INSTNAME
+  declare LINE PRIV_IP STATUS LOCK_USER I IP HOST_NAME ZONE INSTNAME PREEMPTIBLE
  if ! ${USE_CACHE}; then
    COLO_RES_AVAILABILITY=()
    COLO_RES_REQUISITIONED=()
    while read -r LINE; do
-      IFS=$'\v' read -r IP STATUS LOCK_USER INSTNAME <<< "${LINE}"
+      IFS=$'\v' read -r IP STATUS LOCK_USER INSTNAME PREEMPTIBLE <<< "${LINE}"
      I=$(colo_res_index_from_ip "${IP}")
      PRIV_IP="${COLO_RES_IP_PRIV[${I}]}"
      HOST_NAME="${COLO_RES_HOSTNAME[${I}]}"
      ZONE="${COLO_RES_ZONE[${I}]}"
-      COLO_RES_AVAILABILITY+=( "$(echo -e "${HOST_NAME}\v${IP}\v${PRIV_IP}\v${STATUS}\v${ZONE}\v${LOCK_USER}\v${INSTNAME}")" )
+      COLO_RES_AVAILABILITY+=( "$(echo -e "${HOST_NAME}\v${IP}\v${PRIV_IP}\v${STATUS}\v${ZONE}\v${LOCK_USER}\v${INSTNAME}\v${PREEMPTIBLE}")" )
    done < <(colo_node_status_all | sort -t $'\v' -k1)
    COLO_RES_AVAILABILITY_CACHED=true
  fi
@ -142,15 +142,15 @@ __colo_node_status_script() {
                    # the time due to ${SOLANA_LOCK_FILE} not existing and is running from a
                    # subshell where normal redirection doesn't work
  exec 9<"${SOLANA_LOCK_FILE}" && flock -s 9 && . "${SOLANA_LOCK_FILE}" && exec 9>&-
-  echo -e "\${SOLANA_LOCK_USER}\\v\${SOLANA_LOCK_INSTANCENAME}\\vEOL"
+  echo -e "\${SOLANA_LOCK_USER}\\v\${SOLANA_LOCK_INSTANCENAME}\\v\${PREEMPTIBLE}\\vEOL"
  exec 2>&3 # Restore stderr
 EOF
 }

 __colo_node_status_result_normalize() {
-  declare IP RC US BY INSTNAME EOL
+  declare IP RC US BY INSTNAME PREEMPTIBLE EOL
  declare ST="DOWN"
-  IFS=$'\v' read -r IP RC US INSTNAME EOL <<< "${1}"
+  IFS=$'\v' read -r IP RC US INSTNAME PREEMPTIBLE EOL <<< "${1}"
  if [ "${RC}" -eq 0 ]; then
    [[ "${EOL}" = "EOL" ]] || echo "${FUNCNAME[0]}: Unexpected input \"${1}\"" 1>&2
    if [ -n "${US}" ]; then
@ -163,7 +163,7 @@ __colo_node_status_result_normalize() {
      ST="FREE"
    fi
  fi
-  echo -e $"${IP}\v${ST}\v${BY}\v${INSTNAME}"
+  echo -e $"${IP}\v${ST}\v${BY}\v${INSTNAME}\v${PREEMPTIBLE}"
 }

 colo_node_status() {
@ -188,6 +188,7 @@ colo_node_requisition() {
  declare INSTANCE_NAME=${2}
  # shellcheck disable=SC2034
  declare SSH_PRIVATE_KEY="${3}"
+  declare PREEMPTIBLE="${4}"

  declare INDEX
  INDEX=$(colo_res_index_from_ip "${IP}")
@ -196,6 +197,7 @@ colo_node_requisition() {
  colo_instance_run "${IP}" "$(cat <<EOF
 SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE}"
 INSTANCE_NAME="${INSTANCE_NAME}"
+PREEMPTIBLE="${PREEMPTIBLE}"
 SSH_AUTHORIZED_KEYS='$("${__colo_here}"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)'
 SSH_PRIVATE_KEY_TEXT="$(<"${SSH_PRIVATE_KEY}")"
 SSH_PUBLIC_KEY_TEXT="$(<"${SSH_PRIVATE_KEY}.pub")"
@ -238,10 +240,12 @@ colo_machine_types_compatible() {

 colo_node_free() {
  declare IP=${1}
+  declare FORCE_DELETE=${2}
  colo_instance_run "${IP}" "$(cat <<EOF
 SOLANA_LOCK_FILE="${SOLANA_LOCK_FILE}"
 SECONDARY_DISK_MOUNT_POINT="${SECONDARY_DISK_MOUNT_POINT}"
 SSH_AUTHORIZED_KEYS='$("${__colo_here}"/add-datacenter-solana-user-authorized_keys.sh 2> /dev/null)'
+FORCE_DELETE="${FORCE_DELETE}"
 $(<"${__colo_here}"/colo-node-onfree.sh)
 EOF
  )"