Remove archiver and storage program (#9992)

automerge
2020-05-14 18:22:47 -07:00
parent 9ef9969d29
commit eb1acaf927
117 changed files with 383 additions and 7735 deletions
--- a/net/common.sh
+++ b/net/common.sh
@@ -37,9 +37,6 @@ clientIpListZone=()
 blockstreamerIpList=()
 blockstreamerIpListPrivate=()
 blockstreamerIpListZone=()
-archiverIpList=()
-archiverIpListPrivate=()
-archiverIpListZone=()

 buildSshOptions() {
  sshOptions=(
--- a/net/gce.sh
+++ b/net/gce.sh
@@ -16,7 +16,6 @@ gce)
  gpuBootstrapLeaderMachineType="$cpuBootstrapLeaderMachineType --accelerator count=1,type=nvidia-tesla-p100"
  clientMachineType="--custom-cpu 16 --custom-memory 20GB"
  blockstreamerMachineType="--machine-type n1-standard-8"
-  archiverMachineType="--custom-cpu 4 --custom-memory 16GB"
  selfDestructHours=8
  ;;
 ec2)
@@ -31,7 +30,6 @@ ec2)
  gpuBootstrapLeaderMachineType=p2.xlarge
  clientMachineType=c5.2xlarge
  blockstreamerMachineType=m5.4xlarge
-  archiverMachineType=c5.xlarge
  selfDestructHours=0
  ;;
 azure)
@@ -42,7 +40,6 @@ azure)
  gpuBootstrapLeaderMachineType=Standard_NC12
  clientMachineType=Standard_D16s_v3
  blockstreamerMachineType=Standard_D16s_v3
-  archiverMachineType=Standard_D4s_v3
  selfDestructHours=0
  ;;
 colo)
@@ -53,7 +50,6 @@ colo)
  gpuBootstrapLeaderMachineType=1
  clientMachineType=0
  blockstreamerMachineType=0
-  archiverMachineType=0
  selfDestructHours=0
  ;;
 *)
@@ -64,11 +60,9 @@ esac
 prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
 additionalValidatorCount=2
 clientNodeCount=0
-archiverNodeCount=0
 blockstreamer=false
 validatorBootDiskSizeInGb=500
 clientBootDiskSizeInGb=75
-archiverBootDiskSizeInGb=500
 validatorAdditionalDiskSizeInGb=
 externalNodes=false
 failOnValidatorBootupFailure=true
@@ -123,7 +117,6 @@ Manage testnet instances
 create-specific options:
   -n [number]      - Number of additional validators (default: $additionalValidatorCount)
   -c [number]      - Number of client nodes (default: $clientNodeCount)
-   -r [number]      - Number of archiver nodes (default: $archiverNodeCount)
   -u               - Include a Blockstreamer (default: $blockstreamer)
   -P               - Use public network IP addresses (default: $publicNetwork)
   -g               - Enable GPU and automatically set validator machine types to $gpuBootstrapLeaderMachineType
@@ -151,7 +144,7 @@ Manage testnet instances
                      Only supported on GCE.
   --dedicated      - Use dedicated instances for additional validators
                      (by default preemptible instances are used to reduce
-                      cost).  Note that the bootstrap validator, archiver,
+                      cost).  Note that the bootstrap validator,
                      blockstreamer and client nodes are always dedicated.
                      Set this flag on colo to prevent your testnet from being pre-empted by nightly test automation.
   --self-destruct-hours [number]
@@ -252,9 +245,6 @@ while getopts "h?p:Pn:c:r:z:gG:a:d:uxf" opt "${shortArgs[@]}"; do
  c)
    clientNodeCount=$OPTARG
    ;;
-  r)
-    archiverNodeCount=$OPTARG
-    ;;
  z)
    containsZone "$OPTARG" "${zones[@]}" || zones+=("$OPTARG")
    ;;
@@ -607,16 +597,6 @@ EOF
    cloud_ForEachInstance recordInstanceIp true blockstreamerIpList
  }

-  if ! $externalNodes; then
-    echo "archiverIpList=()" >> "$configFile"
-    echo "archiverIpListPrivate=()" >> "$configFile"
-  fi
-  echo "Looking for archiver instances..."
-  cloud_FindInstances "$prefix-archiver"
-  [[ ${#instances[@]} -eq 0 ]] || {
-    cloud_ForEachInstance recordInstanceIp true archiverIpList
-  }
-
  echo "Wrote $configFile"
  $metricsWriteDatapoint "testnet-deploy net-config-complete=1"
 }
@@ -696,7 +676,6 @@ create)
  Bootstrap validator = $bootstrapLeaderMachineType (GPU=$enableGpu)
  Additional validators = $additionalValidatorCount x $validatorMachineType
  Client(s) = $clientNodeCount x $clientMachineType
-  Archivers(s) = $archiverNodeCount x $archiverMachineType
  Blockstreamer = $blockstreamer
 ========================================================================================

@@ -900,12 +879,6 @@ EOF
      "$startupScript" "$blockstreamerAddress" "$bootDiskType" "" "$maybePreemptible" "$sshPrivateKey"
  fi

-  if [[ $archiverNodeCount -gt 0 ]]; then
-    cloud_CreateInstances "$prefix" "$prefix-archiver" "$archiverNodeCount" \
-      false "$archiverMachineType" "${zones[0]}" "$archiverBootDiskSizeInGb" \
-      "$startupScript" "" "" "" "$maybePreemptible" "$sshPrivateKey"
-  fi
-
  $metricsWriteDatapoint "testnet-deploy net-create-complete=1"

  prepareInstancesAndWriteConfigFile
@@ -929,7 +902,6 @@ info)
    echo "NET_NUM_VALIDATORS=${#validatorIpList[@]}"
    echo "NET_NUM_CLIENTS=${#clientIpList[@]}"
    echo "NET_NUM_BLOCKSTREAMERS=${#blockstreamerIpList[@]}"
-    echo "NET_NUM_ARCHIVERS=${#archiverIpList[@]}"
  else
    printNode "Node Type" "Public IP" "Private IP" "Zone"
    echo "-------------------+-----------------+-----------------+--------------"
@@ -976,18 +948,6 @@ info)
    done
  fi

-  if [[ ${#archiverIpList[@]} -gt 0 ]]; then
-    for i in $(seq 0 $(( ${#archiverIpList[@]} - 1)) ); do
-      ipAddress=${archiverIpList[$i]}
-      ipAddressPrivate=${archiverIpListPrivate[$i]}
-      zone=${archiverIpListZone[$i]}
-      if $evalInfo; then
-        echo "NET_ARCHIVER${i}_IP=$ipAddress"
-      else
-        printNode archiver "$ipAddress" "$ipAddressPrivate" "$zone"
-      fi
-    done
-  fi
  ;;
 status)
  cloud_StatusAll
--- a/net/net.sh
+++ b/net/net.sh
@@ -260,7 +260,7 @@ startBootstrapLeader() {
         $deployMethod \
         bootstrap-validator \
         $entrypointIp \
-         $((${#validatorIpList[@]} + ${#blockstreamerIpList[@]} + ${#archiverIpList[@]})) \
+         $((${#validatorIpList[@]} + ${#blockstreamerIpList[@]})) \
         \"$RUST_LOG\" \
         $skipSetup \
         $failOnValidatorBootupFailure \
@@ -329,7 +329,7 @@ startNode() {
         $deployMethod \
         $nodeType \
         $entrypointIp \
-         $((${#validatorIpList[@]} + ${#blockstreamerIpList[@]} + ${#archiverIpList[@]})) \
+         $((${#validatorIpList[@]} + ${#blockstreamerIpList[@]})) \
         \"$RUST_LOG\" \
         $skipSetup \
         $failOnValidatorBootupFailure \
@@ -461,13 +461,10 @@ getNodeType() {
  nodeIndex=0 # <-- global
  nodeType=validator # <-- global

-  for ipAddress in "${validatorIpList[@]}" b "${blockstreamerIpList[@]}" r "${archiverIpList[@]}"; do
+  for ipAddress in "${validatorIpList[@]}" b "${blockstreamerIpList[@]}"; do
    if [[ $ipAddress = b ]]; then
      nodeType=blockstreamer
      continue
-    elif [[ $ipAddress = r ]]; then
-      nodeType=archiver
-      continue
    fi

    if [[ $ipAddress = "$nodeAddress" ]]; then
@@ -543,7 +540,7 @@ deploy() {
  $metricsWriteDatapoint "testnet-deploy net-start-begin=1"

  declare bootstrapLeader=true
-  for nodeAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}" "${archiverIpList[@]}"; do
+  for nodeAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}"; do
    nodeType=
    nodeIndex=
    getNodeType
@@ -623,7 +620,7 @@ deploy() {
  echo
  echo "--- Deployment Successful"
  echo "Bootstrap validator deployment took $bootstrapNodeDeployTime seconds"
-  echo "Additional validator deployment (${#validatorIpList[@]} validators, ${#blockstreamerIpList[@]} blockstreamer nodes, ${#archiverIpList[@]} archivers) took $additionalNodeDeployTime seconds"
+  echo "Additional validator deployment (${#validatorIpList[@]} validators, ${#blockstreamerIpList[@]} blockstreamer nodes) took $additionalNodeDeployTime seconds"
  echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds"
  echo "Network start logs in $netLogDir"
 }
@@ -662,7 +659,7 @@ stop() {

  declare loopCount=0
  pids=()
-  for ipAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}" "${archiverIpList[@]}" "${clientIpList[@]}"; do
+  for ipAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}" "${clientIpList[@]}"; do
    stopNode "$ipAddress" false

    # Stagger additional node stop time to avoid too many concurrent ssh
@@ -1018,9 +1015,6 @@ logs)
  for ipAddress in "${blockstreamerIpList[@]}"; do
    fetchRemoteLog "$ipAddress" validator
  done
-  for ipAddress in "${archiverIpList[@]}"; do
-    fetchRemoteLog "$ipAddress" validator
-  done
  ;;
 netem)
  if [[ -n $netemConfigFile ]]; then
--- a/net/remote/remote-node.sh
+++ b/net/remote/remote-node.sh
@@ -389,31 +389,6 @@ EOF
      multinode-demo/delegate-stake.sh "${args[@]}" "$internalNodesStakeLamports"
    fi
    ;;
-  archiver)
-    if [[ $deployMethod != skip ]]; then
-      net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/
-    fi
-
-    args=(
-      --entrypoint "$entrypointIp:8001"
-    )
-
-    if [[ $airdropsEnabled != true ]]; then
-      # If this ever becomes a problem, we need to provide the `--identity`
-      # argument to an existing system account with lamports in it
-      echo "Error: archivers not supported without airdrops"
-      exit 1
-    fi
-
-cat >> ~/solana/on-reboot <<EOF
-    nohup multinode-demo/archiver.sh ${args[@]} > validator.log.\$now 2>&1 &
-    pid=\$!
-    oom_score_adj "\$pid" 1000
-    disown
-EOF
-    ~/solana/on-reboot
-    sleep 1
-    ;;
  *)
    echo "Error: unknown node type: $nodeType"
    exit 1
--- a/net/ssh.sh
+++ b/net/ssh.sh
@@ -72,15 +72,6 @@ else
  done
 fi
 echo
-echo Archivers:
-if [[ ${#archiverIpList[@]} -eq 0 ]]; then
-  echo "  None"
-else
-  for ipAddress in "${archiverIpList[@]}"; do
-    printNode validator "$ipAddress"
-  done
-fi
-echo
 echo "Use |scp.sh| to transfer files to and from nodes"
 echo