net.sh: Support multiple validators with active stake from the start

multinode-demo/bootstrap-validator.sh

@@ -63,6 +63,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --wait-for-supermajority ]]; then
       args+=("$1" "$2")
       shift 2
+    elif [[ $1 == --expected-bank-hash ]]; then
+      args+=("$1" "$2")
+      shift 2
     else
       echo "Unknown argument: $1"
       $program --help

multinode-demo/validator.sh

@@ -151,6 +151,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --wait-for-supermajority ]]; then
       args+=("$1" "$2")
       shift 2
+    elif [[ $1 == --expected-bank-hash ]]; then
+      args+=("$1" "$2")
+      shift 2
     elif [[ $1 = -h ]]; then
       usage "$@"
     else

net/net.sh

@@ -71,6 +71,10 @@ Operate a configured testnet
                                       - If set, disables the faucet keypair.  Nodes must be funded in genesis config
    --faucet-lamports NUM_LAMPORTS_TO_MINT
                                       - Override the default 500000000000000000 lamports minted in genesis
+   --extra-primordial-stakes NUM_EXTRA_PRIMORDIAL_STAKES
+                                      - Number of extra nodes to be initially staked in genesis.
+                                        Implies --wait-for-supermajority 1 --async-node-init and the supermajority
+                                        wait slot may be overridden with the corresponding flag
    --internal-nodes-stake-lamports NUM_LAMPORTS_PER_NODE
                                       - Amount to stake internal nodes.
    --internal-nodes-lamports NUM_LAMPORTS_PER_NODE
@@ -284,6 +288,7 @@ startBootstrapLeader() {
          \"$gpuMode\" \
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
+         \"$extraPrimordialStakes\" \
       "
 
   ) >> "$logFile" 2>&1 || {
@@ -354,6 +359,7 @@ startNode() {
          \"$gpuMode\" \
          \"$maybeWarpSlot\" \
          \"$waitForNodeInit\" \
+         \"$extraPrimordialStakes\" \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
@@ -759,6 +765,7 @@ clientDelayStart=0
 netLogDir=
 maybeWarpSlot=
 waitForNodeInit=true
+extraPrimordialStakes=0
 
 command=$1
 [[ -n $command ]] || usage
@@ -865,6 +872,15 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --async-node-init ]]; then
       waitForNodeInit=false
       shift 1
+    elif [[ $1 == --extra-primordial-stakes ]]; then
+      extraPrimordialStakes=$2
+      # Extra primoridial stakes require that all of the validators start at
+      # the same time. Force async init and wait for supermajority here.
+      waitForNodeInit=false
+      if [[ -z "$maybeWaitForSupermajority" ]]; then
+        maybeWaitForSupermajority="--wait-for-supermajority 1"
+      fi
+      shift 2
     else
       usage "Unknown long option: $1"
     fi
@@ -983,6 +999,16 @@ else
   fi
 fi
 
+if [[ -n "$maybeWaitForSupermajority" && -n "$maybeWarpSlot" ]]; then
+  read -r _ waitSlot <<<"$maybeWaitForSupermajority"
+  read -r _ warpSlot <<<"$maybeWarpSlot"
+  if [[ $waitSlot -ne $warpSlot ]]; then
+    echo "Error: When specifying both --wait-for-supermajority and --warp-slot,"
+    echo "they must use the same slot. ($waitSlot != $warpSlot)"
+    exit 1
+  fi
+fi
+
 checkPremptibleInstances
 
 case $command in

net/remote/remote-node.sh

@@ -27,6 +27,7 @@ extraNodeArgs="${18}"
 gpuMode="${19:-auto}"
 maybeWarpSlot="${20}"
 waitForNodeInit="${21}"
+extraPrimordialStakes="${22:=0}"
 set +x
 
 missing() {
@@ -142,8 +143,20 @@ EOF
         declare name=$1
         if [[ -f net/keypairs/"$name".json ]]; then
           cp net/keypairs/"$name".json config/"$name".json
+          if [[ "$name" =~ ^validator-identity- ]]; then
+            name="${name//-identity-/-vote-}"
+            cp net/keypairs/"$name".json config/"$name".json
+            name="${name//-vote-/-stake-}"
+            cp net/keypairs/"$name".json config/"$name".json
+          fi
         else
           solana-keygen new --no-passphrase -so config/"$name".json
+          if [[ "$name" =~ ^validator-identity- ]]; then
+            name="${name//-identity-/-vote-}"
+            solana-keygen new --no-passphrase -so config/"$name".json
+            name="${name//-vote-/-stake-}"
+            solana-keygen new --no-passphrase -so config/"$name".json
+          fi
         fi
         if [[ -n $internalNodesLamports ]]; then
           declare pubkey
@@ -212,14 +225,48 @@ EOF
       if [[ -f net/keypairs/bootstrap-validator-identity.json ]]; then
         export BOOTSTRAP_VALIDATOR_IDENTITY_KEYPAIR=net/keypairs/bootstrap-validator-identity.json
       fi
+      if [[ "$extraPrimordialStakes" -gt 0 ]]; then
+        if [[ "$extraPrimordialStakes" -gt "$numNodes" ]]; then
+          echo "warning: extraPrimordialStakes($extraPrimordialStakes) clamped to numNodes($numNodes)"
+          extraPrimordialStakes=$numNodes
+        fi
+        for i in $(seq "$extraPrimordialStakes"); do
+          args+=(--bootstrap-validator "$(solana-keygen pubkey "config/validator-identity-$i.json")"
+                                       "$(solana-keygen pubkey "config/validator-vote-$i.json")"
+                                       "$(solana-keygen pubkey "config/validator-stake-$i.json")"
+          )
+        done
+      fi
+
       multinode-demo/setup.sh "${args[@]}"
 
+      maybeWaitForSupermajority=
+      # shellcheck disable=SC2086 # Do not want to quote $extraNodeArgs
+      set -- $extraNodeArgs
+      while [[ -n $1 ]]; do
+        if [[ $1 = "--wait-for-supermajority" ]]; then
+          maybeWaitForSupermajority=$2
+          break
+        fi
+        shift
+      done
+
+      if [[ -z "$maybeWarpSlot" ]]; then
+        maybeWarpSlot="--warp-slot $maybeWaitForSupermajority"
+      fi
+
       if [[ -n "$maybeWarpSlot" ]]; then
         # shellcheck disable=SC2086 # Do not want to quote $maybeWarSlot
         solana-ledger-tool -l config/bootstrap-validator create-snapshot 0 config/bootstrap-validator $maybeWarpSlot
       fi
 
       solana-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version
+
+      if [[ -n "$maybeWaitForSupermajority" ]]; then
+        bankHash=$(solana-ledger-tool -l config/bootstrap-validator bank-hash)
+        extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash"
+        echo "$bankHash" > config/bank-hash
+      fi
     fi
     args=(
       --gossip-host "$entrypointIp"
@@ -262,10 +309,17 @@ EOF
       else
         net/scripts/rsync-retry.sh -vPrc \
           "$entrypointIp":~/solana/config/validator-identity-"$nodeIndex".json config/validator-identity.json
+        net/scripts/rsync-retry.sh -vPrc \
+          "$entrypointIp":~/solana/config/validator-stake-"$nodeIndex".json config/stake-account.json
+        net/scripts/rsync-retry.sh -vPrc \
+          "$entrypointIp":~/solana/config/validator-vote-"$nodeIndex".json config/vote-account.json
       fi
       net/scripts/rsync-retry.sh -vPrc \
         "$entrypointIp":~/solana/config/shred-version config/shred-version
 
+      net/scripts/rsync-retry.sh -vPrc \
+        "$entrypointIp":~/solana/config/bank-hash config/bank-hash || true
+
       net/scripts/rsync-retry.sh -vPrc \
         "$entrypointIp":~/solana/config/faucet.json config/faucet.json
     fi
@@ -293,11 +347,19 @@ EOF
       solana-keygen new --no-passphrase -so config/validator-identity.json
     fi
     args+=(--identity config/validator-identity.json)
+    if [[ ! -f config/vote-account.json ]]; then
+      solana-keygen new --no-passphrase -so config/vote-account.json
+    fi
+    args+=(--vote-account config/vote-account.json)
 
     if [[ $airdropsEnabled != true ]]; then
       args+=(--no-airdrop)
     fi
 
+    if [[ -r config/bank-hash ]]; then
+      args+=(--expected-bank-hash "$(cat config/bank-hash)")
+    fi
+
     set -x
     # Add the faucet keypair to validators for convenient access from tools
     # like bench-tps and add to blocktreamers to run a faucet
@@ -350,7 +412,9 @@ EOF
         args+=(--keypair config/validator-identity.json)
       fi
 
-      multinode-demo/delegate-stake.sh "${args[@]}" "$internalNodesStakeLamports"
+      if [[ ${#extraPrimordialStakes} -eq 0 ]]; then
+        multinode-demo/delegate-stake.sh "${args[@]}" "$internalNodesStakeLamports"
+      fi
     fi
     ;;
   *)