net/ can now deploy Snaps

net/net.sh

@@ -15,23 +15,60 @@ usage() {
   cat <<EOF
 usage: $0 [start|stop]
 
-Manage a multinode network
+Operate a configured testnet
+
+ start - Start the network
+ stop  - Stop the network
+
+ start-specific options:
+   -S snapFilename     - Deploy the specified Snap file
+   -s edge|beta|stable - Deploy the latest Snap on the specified Snap release channel
+   -a "setup args"     - Optional additional arguments for ./multinode-demo/setup.sh
+
+   Note: if RUST_LOG is set in the environment it will be propogated into the
+         network nodes.
+
+ stop-specific options:
+   none
 
- start|stop    - Start or stop the network
 EOF
   exit $exitcode
 }
 
+snapChannel=
+snapFilename=
+nodeSetupArgs=
+deployMethod=local
+
 command=$1
 [[ -n $command ]] || usage
 shift
 [[ $command = start || $command = stop ]] || usage "Invalid command: $command"
 
-while getopts "h?" opt; do
+while getopts "h?S:s:a:" opt; do
   case $opt in
   h | \?)
     usage
     ;;
+  S)
+    [[ $command = start ]] || usage "-s is only valid with the 'start' command"
+    snapFilename=$OPTARG
+    [[ -f $snapFilename ]] || usage "Snap not readable: $snapFilename"
+    deployMethod=snap
+    ;;
+  s)
+    case $OPTARG in
+    edge|beta|stable)
+      snapChannel=$OPTARG
+      ;;
+    *)
+      usage "Invalid snap channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  a)
+    nodeSetupArgs="$OPTARG"
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -64,15 +101,6 @@ common_start_setup() {
 
   (
     set -x
-
-    ssh "${sshOptions[@]}" "$ipAddress" "
-      set -ex;
-      sudo systemctl disable apt-daily.service # disable run when system boot
-      sudo systemctl disable apt-daily.timer   # disable timer run
-      sudo apt-get --assume-yes install rsync libssl-dev;
-      mkdir -p ~/solana ~/.cargo/bin;
-    "
-
     test -d "$SOLANA_ROOT"
     rsync -vPrz -e "ssh ${sshOptions[*]}" \
       "$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
@@ -88,11 +116,24 @@ startLeader() {
 
   common_start_setup "$ipAddress" "$logFile"
 
+  # Deploy local binaries to leader.  Validators and clients later fetch the
+  # binaries from the leader.
   (
     set -x
-    rsync -vPrz -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress":~/.cargo/bin/
+    case $deployMethod in
+    snap)
+      rsync -vPrz -e "ssh ${sshOptions[*]}" "$snapFilename" "$ipAddress:~/solana/solana.snap"
+      ;;
+    local)
+      rsync -vPrz -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/"
+      ;;
+    *)
+      usage "Internal error: invalid deployMethod: $deployMethod"
+      ;;
+    esac
+
     ssh "${sshOptions[@]}" -f "$ipAddress" \
-      "./solana/net/remote/remote_leader.sh"
+      "./solana/net/remote/remote_node.sh $deployMethod leader $leaderIp \"$nodeSetupArgs\" \"$RUST_LOG\""
   ) >> "$logFile"
 }
 
@@ -106,7 +147,7 @@ startValidator() {
   (
     set -x
     ssh "${sshOptions[@]}" -f "$ipAddress" \
-      "./solana/net/remote/remote_validator.sh $leaderIp"
+      "./solana/net/remote/remote_node.sh $deployMethod validator $leaderIp \"$nodeSetupArgs\" \"$RUST_LOG\""
    ) >> "$logFile"
 }
 
@@ -117,35 +158,67 @@ startClient() {
   echo "Starting client: $leaderIp"
   common_start_setup "$ipAddress" "$logFile"
 
+  declare expectedNodeCount=$((${#validatorIpList[@]} + 1))
+
   ssh "${sshOptions[@]}" -f "$ipAddress" \
-    "./solana/net/remote/remote_client.sh $leaderIp" >> "$logFile"
+    "./solana/net/remote/remote_client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\"" >> "$logFile"
 }
 
 start() {
-  echo "Deployment started at $(date)"
-  SECONDS=0
-  leaderDeployTime=
+  [[ $command = "start" ]] || return
 
+  case $deployMethod in
+  snap)
+    if [[ -n $snapChannel ]]; then
+      if [[ $(uname) != Linux ]]; then
+        echo Error: snap channel deployment only supported in Linux
+        exit 1
+      fi
+      usage "TODO: the snap download command below is probably wrong..."
+      snap download --"$snapChannel" solana
+      snapFilename=solana.snap
+    fi
+    ;;
+  local)
+    build
+    ;;
+  *)
+    usage "Internal error: invalid deployMethod: $deployMethod"
+    ;;
+  esac
+
+  echo "Deployment started at $(date)"
+
+  SECONDS=0
+  declare leaderDeployTime=
+  declare networkVersion=unknown
   startLeader "$leaderIp" "$netLogDir/leader-$leaderIp.log"
   leaderDeployTime=$SECONDS
-  SECONDS=0
 
+  SECONDS=0
   for ipAddress in "${validatorIpList[@]}"; do
     startValidator "$ipAddress" "$netLogDir/validator-$ipAddress.log" &
   done
-
   wait
   validatorDeployTime=$SECONDS
-  SECONDS=0
 
+  SECONDS=0
   for ipAddress in "${clientIpList[@]}"; do
     startClient "$ipAddress" "$netLogDir/client-$ipAddress.log"
   done
-
   clientDeployTime=$SECONDS
-  SECONDS=0
   wait
 
+  if [[ $deployMethod = "snap" ]]; then
+    IFS=\  read -r _ networkVersion _ < <(
+      ssh "${sshOptions[@]}" "$leaderIp" \
+        "snap info solana | grep \"^installed:\""
+    )
+    networkVersion=${networkVersion/0+git./}
+  fi
+
+  $metricsWriteDatapoint "testnet-deploy,name=$netBasename start=1,version=\"$networkVersion\""
+
   echo
   echo "================================================================="
   echo "Deployment finished at $(date)"
@@ -165,7 +238,11 @@ stop_node() {
     set -x
     ssh "${sshOptions[@]}" "$ipAddress" "
       set -x;
-      pkill -9 solana- remote_ oom-monitor
+      if snap list solana; then
+        sudo snap set solana mode=;
+        sudo snap remove solana;
+      fi; \
+      pkill -9 solana- remote_ oom-monitor;
     "
   ) || true
 }
@@ -173,6 +250,8 @@ stop_node() {
 stop() {
   SECONDS=0
 
+  $metricsWriteDatapoint "testnet-deploy,name=$netBasename stop=1"
+
   stop_node "$leaderIp"
 
   for ipAddress in "${validatorIpList[@]}" "${clientIpList[@]}"; do
@@ -182,14 +261,6 @@ stop() {
   echo "Stopping nodes took $SECONDS seconds"
 }
 
-mkdir -p log
+stop
+start
 
-if [[ $command == "start" ]]; then
-  build
-  stop
-  start
-elif [[ $command == "stop" ]]; then
-  stop
-else
-  usage "Unknown command: $command"
-fi