v0.14: various net/ fixes for large clusters (#4080)

* net.sh: Add -F to discard validator nodes that didn't bootup successfully

* Relax sanity node count when validator bootup failure is permitted

* Less sanity for testnet-demo

* net.sh: Add -F to discard validator nodes that didn't bootup successfully

net/net.sh

@@ -51,6 +51,7 @@ Operate a configured testnet
                                         to the bench-tps client.
 
  sanity/start/update-specific options:
+   -F                   - Discard validator nodes that didn't bootup successfully
    -o noLedgerVerify    - Skip ledger verification
    -o noValidatorSanity - Skip fullnode sanity
    -o rejectExtraNodes  - Require the exact number of nodes
@@ -80,12 +81,13 @@ numBenchTpsClients=0
 numBenchExchangeClients=0
 benchTpsExtraArgs=
 benchExchangeExtraArgs=
+failOnValidatorBootupFailure=true
 
 command=$1
 [[ -n $command ]] || usage
 shift
 
-while getopts "h?T:t:o:f:rD:i:c:" opt; do
+while getopts "h?T:t:o:f:rD:i:c:F" opt; do
   case $opt in
   h | \?)
     usage
@@ -167,6 +169,9 @@ while getopts "h?T:t:o:f:rD:i:c:" opt; do
     }
     getClientTypeAndNum
     ;;
+  F)
+    failOnValidatorBootupFailure=false
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -291,6 +296,7 @@ startBootstrapLeader() {
          \"$RUST_LOG\" \
          $skipSetup \
          $leaderRotation \
+         $failOnValidatorBootupFailure \
       "
   ) >> "$logFile" 2>&1 || {
     cat "$logFile"
@@ -319,6 +325,7 @@ startNode() {
          \"$RUST_LOG\" \
          $skipSetup \
          $leaderRotation \
+         $failOnValidatorBootupFailure \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
@@ -475,9 +482,13 @@ start() {
     declare ok=true
     wait "$pid" || ok=false
     if ! $ok; then
+      echo "+++ fullnode failed to start"
       cat "$netLogDir/fullnode-$pid.log"
-      echo ^^^ +++
-      exit 1
+      if $failOnValidatorBootupFailure; then
+        exit 1
+      else
+        echo "Failure is non-fatal"
+      fi
     fi
   done