Private IP networks now work, and are the default

2018-09-04 22:21:58 -07:00
parent db9219ccc8
commit e677cda027
6 changed files with 78 additions and 61 deletions
--- a/net/common.sh
+++ b/net/common.sh
@ -16,12 +16,14 @@ source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh

 configFile="$netConfigDir/config"

-clientIpList=()
+entrypointIp=
+publicNetwork=
 leaderIp=
 netBasename=
-sshOptions=()
 sshPrivateKey=
 sshUsername=
+clientIpList=()
+sshOptions=()
 validatorIpList=()

 buildSshOptions() {
@ -40,11 +42,13 @@ loadConfigFile() {

  # shellcheck source=/dev/null
  source "$configFile"
-  [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
+  [[ -n "$entrypointIp" ]] || usage "Config file invalid, entrypointIp unspecified: $configFile"
+  [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile"
  [[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile"
-  [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
-  [[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile"
+  [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
  [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile"
+  [[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile"
+  [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"

  buildSshOptions
  configureMetrics
--- a/net/gce.sh
+++ b/net/gce.sh
@ -17,7 +17,7 @@ clientMachineType=n1-standard-16
 clientAccelerator=

 imageName="ubuntu-16-04-cuda-9-2-new"
-internalNetwork=false
+publicNetwork=false
 zone="us-west1-b"

 usage() {
@ -42,7 +42,7 @@ Configure a GCE-based testnet
 create-specific options:
   -n number        - Number of validator nodes (default: $validatorNodeCount)
   -c number        - Number of client nodes (default: $clientNodeCount)
-   -P               - Use GCE internal/private network (default: $internalNetwork)
+   -P               - Use public network IP addresses (default: $publicNetwork)
   -z               - GCP Zone for the nodes (default: $zone)
   -i imageName     - Existing image on GCE (default: $imageName)
   -g               - Enable GPU
@ -72,7 +72,7 @@ while getopts "h?p:Pi:n:c:z:g" opt; do
    prefix=$OPTARG
    ;;
  P)
-    internalNetwork=true
+    publicNetwork=true
    ;;
  i)
    imageName=$OPTARG
@ -97,9 +97,12 @@ done


 prepareInstancesAndWriteConfigFile() {
-  echo "# autogenerated at $(date)" >> "$configFile"

-  echo "netBasename=$prefix" >> "$configFile"
+  cat >> "$configFile" <<EOF
+# autogenerated at $(date)
+netBasename=$prefix
+publicNetwork=$publicNetwork
+EOF

  declare sshPrivateKey="$netConfigDir/id_$prefix"
  rm -rf "$sshPrivateKey"{,.pub}
@ -116,10 +119,13 @@ prepareInstancesAndWriteConfigFile() {

    declare arrayName="$6"

-    if $internalNetwork; then
-      echo "$arrayName+=($privateIp) # $name" >> "$configFile"
-    else
-      echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
+    echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
+    if [[ $arrayName = "leaderIp" ]]; then
+      if $publicNetwork; then
+        echo "entrypointIp=$publicIp" >> "$configFile"
+      else
+        echo "entrypointIp=$privateIp" >> "$configFile"
+      fi
    fi
  }

--- a/net/net.sh
+++ b/net/net.sh
@ -25,7 +25,6 @@ Operate a configured testnet
 start-specific options:
   -S snapFilename      - Deploy the specified Snap file
   -s edge|beta|stable  - Deploy the latest Snap on the specified Snap release channel
-   -a "setup args"      - Optional additional arguments for ./multinode-demo/setup.sh

   Note: if RUST_LOG is set in the environment it will be propogated into the
         network nodes.
@ -43,7 +42,6 @@ EOF

 snapChannel=
 snapFilename=
-nodeSetupArgs=
 deployMethod=local
 sanityExtraArgs=

@ -51,7 +49,7 @@ command=$1
 [[ -n $command ]] || usage
 shift

-while getopts "h?S:s:a:o:" opt; do
+while getopts "h?S:s:o:" opt; do
  case $opt in
  h | \?)
    usage
@ -72,9 +70,6 @@ while getopts "h?S:s:a:o:" opt; do
      ;;
    esac
    ;;
-  a)
-    nodeSetupArgs="$OPTARG"
-    ;;
  o)
    case $OPTARG in
    noLedgerVerify|noValidatorSanity)
@ -144,7 +139,7 @@ startLeader() {
    esac

    ssh "${sshOptions[@]}" -n "$ipAddress" \
-      "./solana/net/remote/remote-node.sh $deployMethod leader $leaderIp $expectedNodeCount \"$nodeSetupArgs\" \"$RUST_LOG\""
+      "./solana/net/remote/remote-node.sh $deployMethod leader $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
  ) >> "$logFile" 2>&1 || {
    cat "$logFile"
    echo "^^^ +++"
@ -161,7 +156,7 @@ startValidator() {
    set -x
    common_start_setup "$ipAddress"
    ssh "${sshOptions[@]}" -n "$ipAddress" \
-      "./solana/net/remote/remote-node.sh $deployMethod validator $leaderIp $expectedNodeCount \"$nodeSetupArgs\" \"$RUST_LOG\""
+      "./solana/net/remote/remote-node.sh $deployMethod validator $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
  ) >> "$netLogDir/validator-$ipAddress.log" 2>&1 &
  declare pid=$!
  ln -sfT "validator-$ipAddress.log" "$netLogDir/validator-$pid.log"
@ -177,7 +172,7 @@ startClient() {
    set -x
    common_start_setup "$ipAddress"
    ssh "${sshOptions[@]}" -f "$ipAddress" \
-      "./solana/net/remote/remote-client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\""
+      "./solana/net/remote/remote-client.sh $deployMethod $entrypointIp $expectedNodeCount \"$RUST_LOG\""
  ) >> "$logFile" 2>&1 || {
    cat "$logFile"
    echo "^^^ +++"
--- a/net/remote/remote-client.sh
+++ b/net/remote/remote-client.sh
@ -3,11 +3,12 @@
 cd "$(dirname "$0")"/../..

 deployMethod="$1"
-leaderIp="$2"
+entrypointIp="$2"
 numNodes="$3"
 RUST_LOG="$4"
+
 [[ -n $deployMethod ]] || exit
-[[ -n $leaderIp ]] || exit
+[[ -n $entrypointIp ]] || exit
 [[ -n $numNodes ]] || exit

 source net/common.sh
@ -22,12 +23,12 @@ scripts/install-earlyoom.sh

 case $deployMethod in
 snap)
-  rsync -vPrc "$leaderIp:~/solana/solana.snap" .
+  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
  sudo snap install solana.snap --devmode --dangerous
  rm solana.snap

  nodeConfig="\
-    leader-ip=$leaderIp \
+    leader-ip=$entrypointIp \
    default-metrics-rate=1 \
    metrics-config=$SOLANA_METRICS_CONFIG \
    rust-log=$RUST_LOG \
@ -43,8 +44,8 @@ local)
  export SOLANA_DEFAULT_METRICS_RATE=1
  export RUST_LOG

-  rsync -vPrc "$leaderIp:~/.cargo/bin/solana*" ~/.cargo/bin/
-  solana_bench_tps="multinode-demo/client.sh $leaderIp:~/solana"
+  rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+  solana_bench_tps="multinode-demo/client.sh $entrypointIp:~/solana"
  ;;
 *)
  echo "Unknown deployment method: $deployMethod"
--- a/net/remote/remote-node.sh
+++ b/net/remote/remote-node.sh
@ -4,35 +4,43 @@ cd "$(dirname "$0")"/../..

 deployMethod="$1"
 nodeType="$2"
-leaderIp="$3"
-numNodes="$4"
-setupArgs="$5"
+publicNetwork="$3"
+entrypointIp="$4"
+numNodes="$5"
 RUST_LOG="$6"

-cat > deployConfig <<EOF
-deployMethod="$deployMethod"
-leaderIp="$leaderIp"
-numNodes="$numNodes"
-EOF
-
 [[ -n $deployMethod ]] || exit
 [[ -n $nodeType ]] || exit
-[[ -n $leaderIp ]] || exit
+[[ -n $publicNetwork ]] || exit
+[[ -n $entrypointIp ]] || exit
+[[ -n $numNodes ]] || exit
+
+cat > deployConfig <<EOF
+deployMethod="$deployMethod"
+entrypointIp="$entrypointIp"
+numNodes="$numNodes"
+EOF

 source net/common.sh
 loadConfigFile

 scripts/install-earlyoom.sh

+if [[ $publicNetwork = true ]]; then
+  setupArgs="-p"
+else
+  setupArgs="-l"
+fi
+

 case $deployMethod in
 snap)
  SECONDS=0
-  rsync -vPrc "$leaderIp:~/solana/solana.snap" .
+  rsync -vPrc "$entrypointIp:~/solana/solana.snap" .
  sudo snap install solana.snap --devmode --dangerous

  commonNodeConfig="\
-    leader-ip=$leaderIp \
+    leader-ip=$entrypointIp \
    default-metrics-rate=1 \
    metrics-config=$SOLANA_METRICS_CONFIG \
    rust-log=$RUST_LOG \
@ -76,17 +84,15 @@ local)

  case $nodeType in
  leader)
-    # shellcheck disable=SC2086 # Don't want to double quote "$setupArgs"
-    ./multinode-demo/setup.sh -t leader -p $setupArgs
+    ./multinode-demo/setup.sh -t leader $setupArgs
    ./multinode-demo/drone.sh > drone.log 2>&1 &
    ./multinode-demo/leader.sh > leader.log 2>&1 &
    ;;
  validator)
-    rsync -vPrc "$leaderIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+    rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/

-    # shellcheck disable=SC2086 # Don't want to double quote "$setupArgs"
-    ./multinode-demo/setup.sh -t validator -p $setupArgs
-    ./multinode-demo/validator.sh "$leaderIp":~/solana "$leaderIp" >validator.log 2>&1 &
+    ./multinode-demo/setup.sh -t validator $setupArgs
+    ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp" >validator.log 2>&1 &
    ;;
  *)
    echo "Error: unknown node type: $nodeType"
--- a/net/remote/remote-sanity.sh
+++ b/net/remote/remote-sanity.sh
@ -3,27 +3,32 @@
 cd "$(dirname "$0")"/../..

 deployMethod=
-leaderIp=
+entrypointIp=
 numNodes=
-# shellcheck source=/dev/null # deployConfig is written by remote_sanity.sh
+
+[[ -r deployConfig ]] || {
+  echo deployConfig missing
+  exit 1
+}
+# shellcheck source=/dev/null # deployConfig is written by remote-node.sh
 source deployConfig

 [[ -n $deployMethod ]] || {
-  echo "deployMethod empty"
+  echo deployMethod empty
  exit 1
 }
-[[ -n $leaderIp ]] || {
-  echo "leaderIp empty"
+[[ -n $entrypointIp ]] || {
+  echo entrypointIp empty
  exit 1
 }
 [[ -n $numNodes ]] || {
-  echo "numNodes empty"
+  echo numNodes empty
  exit 1
 }

 ledgerVerify=true
 validatorSanity=true
-while [[ $1 = "-o" ]]; do
+while [[ $1 = -o ]]; do
  opt="$2"
  shift 2
  case $opt in
@ -56,7 +61,7 @@ local)
  PATH="$HOME"/.cargo/bin:"$PATH"
  export USE_INSTALL=1

-  solana_bench_tps="multinode-demo/client.sh $leaderIp:~/solana"
+  solana_bench_tps="multinode-demo/client.sh $entrypointIp:~/solana"
  solana_ledger_tool=solana-ledger-tool
  ledger=config/ledger
  ;;
@ -66,19 +71,19 @@ local)
 esac


-echo "--- $leaderIp: wallet sanity"
+echo "--- $entrypointIp: wallet sanity"
 (
  set -x
-  multinode-demo/test/wallet-sanity.sh "$leaderIp"
+  multinode-demo/test/wallet-sanity.sh "$entrypointIp"
 )

-echo "--- $leaderIp: node count"
+echo "--- $entrypointIp: node count"
 (
  set -x
  $solana_bench_tps --num-nodes "$numNodes" --converge-only
 )

-echo "--- $leaderIp: verify ledger"
+echo "--- $entrypointIp: verify ledger"
 if $ledgerVerify; then
  if [[ -d $ledger ]]; then
    (
@ -97,12 +102,12 @@ else
 fi


-echo "--- $leaderIp: validator sanity"
+echo "--- $entrypointIp: validator sanity"
 if $validatorSanity; then
  (
    set -ex -o pipefail
    ./multinode-demo/setup.sh -t validator
-    timeout 10s ./multinode-demo/validator.sh "$leaderIp" "$leaderIp:8001" 2>&1 | tee validator.log
+    timeout 10s ./multinode-demo/validator.sh "$entrypointIp" "$entrypointIp:8001" 2>&1 | tee validator.log
  ) || {
    exitcode=$?
    [[ $exitcode -eq 124 ]] || exit $exitcode