Private IP networks now work, and are the default

net/common.sh

@@ -16,12 +16,14 @@ source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
 
 configFile="$netConfigDir/config"
 
-clientIpList=()
+entrypointIp=
+publicNetwork=
 leaderIp=
 netBasename=
-sshOptions=()
 sshPrivateKey=
 sshUsername=
+clientIpList=()
+sshOptions=()
 validatorIpList=()
 
 buildSshOptions() {
@@ -40,11 +42,13 @@ loadConfigFile() {
 
   # shellcheck source=/dev/null
   source "$configFile"
-  [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
+  [[ -n "$entrypointIp" ]] || usage "Config file invalid, entrypointIp unspecified: $configFile"
+  [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile"
   [[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile"
-  [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
+  [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
-  [[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile"
   [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile"
+  [[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile"
+  [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
 
   buildSshOptions
   configureMetrics

net/gce.sh

@@ -17,7 +17,7 @@ clientMachineType=n1-standard-16
 clientAccelerator=
 
 imageName="ubuntu-16-04-cuda-9-2-new"
-internalNetwork=false
+publicNetwork=false
 zone="us-west1-b"
 
 usage() {
@@ -42,7 +42,7 @@ Configure a GCE-based testnet
  create-specific options:
    -n number        - Number of validator nodes (default: $validatorNodeCount)
    -c number        - Number of client nodes (default: $clientNodeCount)
-   -P               - Use GCE internal/private network (default: $internalNetwork)
+   -P               - Use public network IP addresses (default: $publicNetwork)
    -z               - GCP Zone for the nodes (default: $zone)
    -i imageName     - Existing image on GCE (default: $imageName)
    -g               - Enable GPU
@@ -72,7 +72,7 @@ while getopts "h?p:Pi:n:c:z:g" opt; do
     prefix=$OPTARG
     ;;
   P)
-    internalNetwork=true
+    publicNetwork=true
     ;;
   i)
     imageName=$OPTARG
@@ -97,9 +97,12 @@ done
 
 
 prepareInstancesAndWriteConfigFile() {
-  echo "# autogenerated at $(date)" >> "$configFile"
 
-  echo "netBasename=$prefix" >> "$configFile"
+  cat >> "$configFile" <<EOF
+# autogenerated at $(date)
+netBasename=$prefix
+publicNetwork=$publicNetwork
+EOF
 
   declare sshPrivateKey="$netConfigDir/id_$prefix"
   rm -rf "$sshPrivateKey"{,.pub}
@@ -116,10 +119,13 @@ prepareInstancesAndWriteConfigFile() {
 
     declare arrayName="$6"
 
-    if $internalNetwork; then
+    echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
-      echo "$arrayName+=($privateIp) # $name" >> "$configFile"
+    if [[ $arrayName = "leaderIp" ]]; then
-    else
+      if $publicNetwork; then
-      echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
+        echo "entrypointIp=$publicIp" >> "$configFile"
+      else
+        echo "entrypointIp=$privateIp" >> "$configFile"
+      fi
     fi
   }
 

net/net.sh

@@ -25,7 +25,6 @@ Operate a configured testnet
  start-specific options:
    -S snapFilename      - Deploy the specified Snap file
    -s edge|beta|stable  - Deploy the latest Snap on the specified Snap release channel
-   -a "setup args"      - Optional additional arguments for ./multinode-demo/setup.sh
 
    Note: if RUST_LOG is set in the environment it will be propogated into the
          network nodes.
@@ -43,7 +42,6 @@ EOF
 
 snapChannel=
 snapFilename=
-nodeSetupArgs=
 deployMethod=local
 sanityExtraArgs=
 
@@ -51,7 +49,7 @@ command=$1
 [[ -n $command ]] || usage
 shift
 
-while getopts "h?S:s:a:o:" opt; do
+while getopts "h?S:s:o:" opt; do
   case $opt in
   h | \?)
     usage
@@ -72,9 +70,6 @@ while getopts "h?S:s:a:o:" opt; do
       ;;
     esac
     ;;
-  a)
-    nodeSetupArgs="$OPTARG"
-    ;;
   o)
     case $OPTARG in
     noLedgerVerify|noValidatorSanity)
@@ -144,7 +139,7 @@ startLeader() {
     esac
 
     ssh "${sshOptions[@]}" -n "$ipAddress" \
-      "./solana/net/remote/remote-node.sh $deployMethod leader $leaderIp $expectedNodeCount \"$nodeSetupArgs\" \"$RUST_LOG\""
+      "./solana/net/remote/remote-node.sh $deployMethod leader $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
   ) >> "$logFile" 2>&1 || {
     cat "$logFile"
     echo "^^^ +++"
@@ -161,7 +156,7 @@ startValidator() {
     set -x
     common_start_setup "$ipAddress"
     ssh "${sshOptions[@]}" -n "$ipAddress" \
-      "./solana/net/remote/remote-node.sh $deployMethod validator $leaderIp $expectedNodeCount \"$nodeSetupArgs\" \"$RUST_LOG\""
+      "./solana/net/remote/remote-node.sh $deployMethod validator $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
   ) >> "$netLogDir/validator-$ipAddress.log" 2>&1 &
   declare pid=$!
   ln -sfT "validator-$ipAddress.log" "$netLogDir/validator-$pid.log"
@@ -177,7 +172,7 @@ startClient() {
     set -x
     common_start_setup "$ipAddress"
     ssh "${sshOptions[@]}" -f "$ipAddress" \
-      "./solana/net/remote/remote-client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\""
+      "./solana/net/remote/remote-client.sh $deployMethod $entrypointIp $expectedNodeCount \"$RUST_LOG\""
   ) >> "$logFile" 2>&1 || {
     cat "$logFile"
     echo "^^^ +++"

net/remote/remote-client.sh

@@ -3,11 +3,12 @@
 cd "$(dirname "$0")"/../..
 
 deployMethod="$1"
-leaderIp="$2"
+entrypointIp="$2"
 numNodes="$3"
 RUST_LOG="$4"
+
 [[ -n $deployMethod ]] || exit
-[[ -n $leaderIp ]] || exit
+[[ -n $entrypointIp ]] || exit
 [[ -n $numNodes ]] || exit
 
 source net/common.sh
@@ -22,12 +23,12 @@ scripts/install-earlyoom.sh
 
 case $deployMethod in
 snap)
-  rsync -vPrc "$leaderIp:~/solana/solana.snap" .
+  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
   sudo snap install solana.snap --devmode --dangerous
   rm solana.snap
 
   nodeConfig="\
-    leader-ip=$leaderIp \
+    leader-ip=$entrypointIp \
     default-metrics-rate=1 \
     metrics-config=$SOLANA_METRICS_CONFIG \
     rust-log=$RUST_LOG \
@@ -43,8 +44,8 @@ local)
   export SOLANA_DEFAULT_METRICS_RATE=1
   export RUST_LOG
 
-  rsync -vPrc "$leaderIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+  rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
-  solana_bench_tps="multinode-demo/client.sh $leaderIp:~/solana"
+  solana_bench_tps="multinode-demo/client.sh $entrypointIp:~/solana"
   ;;
 *)
   echo "Unknown deployment method: $deployMethod"

net/remote/remote-node.sh

@@ -4,35 +4,43 @@ cd "$(dirname "$0")"/../..
 
 deployMethod="$1"
 nodeType="$2"
-leaderIp="$3"
+publicNetwork="$3"
-numNodes="$4"
+entrypointIp="$4"
-setupArgs="$5"
+numNodes="$5"
 RUST_LOG="$6"
 
-cat > deployConfig <<EOF
-deployMethod="$deployMethod"
-leaderIp="$leaderIp"
-numNodes="$numNodes"
-EOF
-
 [[ -n $deployMethod ]] || exit
 [[ -n $nodeType ]] || exit
-[[ -n $leaderIp ]] || exit
+[[ -n $publicNetwork ]] || exit
+[[ -n $entrypointIp ]] || exit
+[[ -n $numNodes ]] || exit
+
+cat > deployConfig <<EOF
+deployMethod="$deployMethod"
+entrypointIp="$entrypointIp"
+numNodes="$numNodes"
+EOF
 
 source net/common.sh
 loadConfigFile
 
 scripts/install-earlyoom.sh
 
+if [[ $publicNetwork = true ]]; then
+  setupArgs="-p"
+else
+  setupArgs="-l"
+fi
+
 
 case $deployMethod in
 snap)
   SECONDS=0
-  rsync -vPrc "$leaderIp:~/solana/solana.snap" .
+  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
   sudo snap install solana.snap --devmode --dangerous
 
   commonNodeConfig="\
-    leader-ip=$leaderIp \
+    leader-ip=$entrypointIp \
     default-metrics-rate=1 \
     metrics-config=$SOLANA_METRICS_CONFIG \
     rust-log=$RUST_LOG \
@@ -76,17 +84,15 @@ local)
 
   case $nodeType in
   leader)
-    # shellcheck disable=SC2086 # Don't want to double quote "$setupArgs"
+    ./multinode-demo/setup.sh -t leader $setupArgs
-    ./multinode-demo/setup.sh -t leader -p $setupArgs
     ./multinode-demo/drone.sh > drone.log 2>&1 &
     ./multinode-demo/leader.sh > leader.log 2>&1 &
     ;;
   validator)
-    rsync -vPrc "$leaderIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+    rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
 
-    # shellcheck disable=SC2086 # Don't want to double quote "$setupArgs"
+    ./multinode-demo/setup.sh -t validator $setupArgs
-    ./multinode-demo/setup.sh -t validator -p $setupArgs
+    ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp" >validator.log 2>&1 &
-    ./multinode-demo/validator.sh "$leaderIp":~/solana "$leaderIp" >validator.log 2>&1 &
     ;;
   *)
     echo "Error: unknown node type: $nodeType"

net/remote/remote-sanity.sh

@@ -3,27 +3,32 @@
 cd "$(dirname "$0")"/../..
 
 deployMethod=
-leaderIp=
+entrypointIp=
 numNodes=
-# shellcheck source=/dev/null # deployConfig is written by remote_sanity.sh
+
+[[ -r deployConfig ]] || {
+  echo deployConfig missing
+  exit 1
+}
+# shellcheck source=/dev/null # deployConfig is written by remote-node.sh
 source deployConfig
 
 [[ -n $deployMethod ]] || {
-  echo "deployMethod empty"
+  echo deployMethod empty
   exit 1
 }
-[[ -n $leaderIp ]] || {
+[[ -n $entrypointIp ]] || {
-  echo "leaderIp empty"
+  echo entrypointIp empty
   exit 1
 }
 [[ -n $numNodes ]] || {
-  echo "numNodes empty"
+  echo numNodes empty
   exit 1
 }
 
 ledgerVerify=true
 validatorSanity=true
-while [[ $1 = "-o" ]]; do
+while [[ $1 = -o ]]; do
   opt="$2"
   shift 2
   case $opt in
@@ -56,7 +61,7 @@ local)
   PATH="$HOME"/.cargo/bin:"$PATH"
   export USE_INSTALL=1
 
-  solana_bench_tps="multinode-demo/client.sh $leaderIp:~/solana"
+  solana_bench_tps="multinode-demo/client.sh $entrypointIp:~/solana"
   solana_ledger_tool=solana-ledger-tool
   ledger=config/ledger
   ;;
@@ -66,19 +71,19 @@ local)
 esac
 
 
-echo "--- $leaderIp: wallet sanity"
+echo "--- $entrypointIp: wallet sanity"
 (
   set -x
-  multinode-demo/test/wallet-sanity.sh "$leaderIp"
+  multinode-demo/test/wallet-sanity.sh "$entrypointIp"
 )
 
-echo "--- $leaderIp: node count"
+echo "--- $entrypointIp: node count"
 (
   set -x
   $solana_bench_tps --num-nodes "$numNodes" --converge-only
 )
 
-echo "--- $leaderIp: verify ledger"
+echo "--- $entrypointIp: verify ledger"
 if $ledgerVerify; then
   if [[ -d $ledger ]]; then
     (
@@ -97,12 +102,12 @@ else
 fi
 
 
-echo "--- $leaderIp: validator sanity"
+echo "--- $entrypointIp: validator sanity"
 if $validatorSanity; then
   (
     set -ex -o pipefail
     ./multinode-demo/setup.sh -t validator
-    timeout 10s ./multinode-demo/validator.sh "$leaderIp" "$leaderIp:8001" 2>&1 | tee validator.log
+    timeout 10s ./multinode-demo/validator.sh "$entrypointIp" "$entrypointIp:8001" 2>&1 | tee validator.log
   ) || {
     exitcode=$?
     [[ $exitcode -eq 124 ]] || exit $exitcode