From e677cda027e72143e232dec04030c395145e17b0 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Tue, 4 Sep 2018 22:21:58 -0700 Subject: [PATCH] Private IP networks now work, and are the default --- net/common.sh | 14 ++++++++----- net/gce.sh | 24 +++++++++++++-------- net/net.sh | 13 ++++-------- net/remote/remote-client.sh | 13 ++++++------ net/remote/remote-node.sh | 42 +++++++++++++++++++++---------------- net/remote/remote-sanity.sh | 33 ++++++++++++++++------------- 6 files changed, 78 insertions(+), 61 deletions(-) diff --git a/net/common.sh b/net/common.sh index 96423e0a04..c220ef3291 100644 --- a/net/common.sh +++ b/net/common.sh @@ -16,12 +16,14 @@ source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh configFile="$netConfigDir/config" -clientIpList=() +entrypointIp= +publicNetwork= leaderIp= netBasename= -sshOptions=() sshPrivateKey= sshUsername= +clientIpList=() +sshOptions=() validatorIpList=() buildSshOptions() { @@ -40,11 +42,13 @@ loadConfigFile() { # shellcheck source=/dev/null source "$configFile" - [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile" + [[ -n "$entrypointIp" ]] || usage "Config file invalid, entrypointIp unspecified: $configFile" + [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile" [[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile" - [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile" - [[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile" + [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile" [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile" + [[ -n $sshUsername ]] || usage "Config file invalid, sshUsername unspecified: $configFile" + [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile" buildSshOptions configureMetrics diff --git a/net/gce.sh b/net/gce.sh index 1dd595ee03..02a5153f21 100755 --- a/net/gce.sh +++ b/net/gce.sh @@ -17,7 +17,7 @@ clientMachineType=n1-standard-16 clientAccelerator= imageName="ubuntu-16-04-cuda-9-2-new" -internalNetwork=false +publicNetwork=false zone="us-west1-b" usage() { @@ -42,7 +42,7 @@ Configure a GCE-based testnet create-specific options: -n number - Number of validator nodes (default: $validatorNodeCount) -c number - Number of client nodes (default: $clientNodeCount) - -P - Use GCE internal/private network (default: $internalNetwork) + -P - Use public network IP addresses (default: $publicNetwork) -z - GCP Zone for the nodes (default: $zone) -i imageName - Existing image on GCE (default: $imageName) -g - Enable GPU @@ -72,7 +72,7 @@ while getopts "h?p:Pi:n:c:z:g" opt; do prefix=$OPTARG ;; P) - internalNetwork=true + publicNetwork=true ;; i) imageName=$OPTARG @@ -97,9 +97,12 @@ done prepareInstancesAndWriteConfigFile() { - echo "# autogenerated at $(date)" >> "$configFile" - echo "netBasename=$prefix" >> "$configFile" + cat >> "$configFile" <> "$configFile" - else - echo "$arrayName+=($publicIp) # $name" >> "$configFile" + echo "$arrayName+=($publicIp) # $name" >> "$configFile" + if [[ $arrayName = "leaderIp" ]]; then + if $publicNetwork; then + echo "entrypointIp=$publicIp" >> "$configFile" + else + echo "entrypointIp=$privateIp" >> "$configFile" + fi fi } diff --git a/net/net.sh b/net/net.sh index 8c258e0600..db87f314c9 100755 --- a/net/net.sh +++ b/net/net.sh @@ -25,7 +25,6 @@ Operate a configured testnet start-specific options: -S snapFilename - Deploy the specified Snap file -s edge|beta|stable - Deploy the latest Snap on the specified Snap release channel - -a "setup args" - Optional additional arguments for ./multinode-demo/setup.sh Note: if RUST_LOG is set in the environment it will be propogated into the network nodes. @@ -43,7 +42,6 @@ EOF snapChannel= snapFilename= -nodeSetupArgs= deployMethod=local sanityExtraArgs= @@ -51,7 +49,7 @@ command=$1 [[ -n $command ]] || usage shift -while getopts "h?S:s:a:o:" opt; do +while getopts "h?S:s:o:" opt; do case $opt in h | \?) usage @@ -72,9 +70,6 @@ while getopts "h?S:s:a:o:" opt; do ;; esac ;; - a) - nodeSetupArgs="$OPTARG" - ;; o) case $OPTARG in noLedgerVerify|noValidatorSanity) @@ -144,7 +139,7 @@ startLeader() { esac ssh "${sshOptions[@]}" -n "$ipAddress" \ - "./solana/net/remote/remote-node.sh $deployMethod leader $leaderIp $expectedNodeCount \"$nodeSetupArgs\" \"$RUST_LOG\"" + "./solana/net/remote/remote-node.sh $deployMethod leader $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\"" ) >> "$logFile" 2>&1 || { cat "$logFile" echo "^^^ +++" @@ -161,7 +156,7 @@ startValidator() { set -x common_start_setup "$ipAddress" ssh "${sshOptions[@]}" -n "$ipAddress" \ - "./solana/net/remote/remote-node.sh $deployMethod validator $leaderIp $expectedNodeCount \"$nodeSetupArgs\" \"$RUST_LOG\"" + "./solana/net/remote/remote-node.sh $deployMethod validator $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\"" ) >> "$netLogDir/validator-$ipAddress.log" 2>&1 & declare pid=$! ln -sfT "validator-$ipAddress.log" "$netLogDir/validator-$pid.log" @@ -177,7 +172,7 @@ startClient() { set -x common_start_setup "$ipAddress" ssh "${sshOptions[@]}" -f "$ipAddress" \ - "./solana/net/remote/remote-client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\"" + "./solana/net/remote/remote-client.sh $deployMethod $entrypointIp $expectedNodeCount \"$RUST_LOG\"" ) >> "$logFile" 2>&1 || { cat "$logFile" echo "^^^ +++" diff --git a/net/remote/remote-client.sh b/net/remote/remote-client.sh index 78ebc485c1..38a653f5e3 100755 --- a/net/remote/remote-client.sh +++ b/net/remote/remote-client.sh @@ -3,11 +3,12 @@ cd "$(dirname "$0")"/../.. deployMethod="$1" -leaderIp="$2" +entrypointIp="$2" numNodes="$3" RUST_LOG="$4" + [[ -n $deployMethod ]] || exit -[[ -n $leaderIp ]] || exit +[[ -n $entrypointIp ]] || exit [[ -n $numNodes ]] || exit source net/common.sh @@ -22,12 +23,12 @@ scripts/install-earlyoom.sh case $deployMethod in snap) - rsync -vPrc "$leaderIp:~/solana/solana.snap" . + rsync -vPrc "$entrypointIp:~/solana/solana.snap" . sudo snap install solana.snap --devmode --dangerous rm solana.snap nodeConfig="\ - leader-ip=$leaderIp \ + leader-ip=$entrypointIp \ default-metrics-rate=1 \ metrics-config=$SOLANA_METRICS_CONFIG \ rust-log=$RUST_LOG \ @@ -43,8 +44,8 @@ local) export SOLANA_DEFAULT_METRICS_RATE=1 export RUST_LOG - rsync -vPrc "$leaderIp:~/.cargo/bin/solana*" ~/.cargo/bin/ - solana_bench_tps="multinode-demo/client.sh $leaderIp:~/solana" + rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/ + solana_bench_tps="multinode-demo/client.sh $entrypointIp:~/solana" ;; *) echo "Unknown deployment method: $deployMethod" diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 246f143d46..2b4c1f00d2 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -4,35 +4,43 @@ cd "$(dirname "$0")"/../.. deployMethod="$1" nodeType="$2" -leaderIp="$3" -numNodes="$4" -setupArgs="$5" +publicNetwork="$3" +entrypointIp="$4" +numNodes="$5" RUST_LOG="$6" -cat > deployConfig < deployConfig < drone.log 2>&1 & ./multinode-demo/leader.sh > leader.log 2>&1 & ;; validator) - rsync -vPrc "$leaderIp:~/.cargo/bin/solana*" ~/.cargo/bin/ + rsync -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/ - # shellcheck disable=SC2086 # Don't want to double quote "$setupArgs" - ./multinode-demo/setup.sh -t validator -p $setupArgs - ./multinode-demo/validator.sh "$leaderIp":~/solana "$leaderIp" >validator.log 2>&1 & + ./multinode-demo/setup.sh -t validator $setupArgs + ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp" >validator.log 2>&1 & ;; *) echo "Error: unknown node type: $nodeType" diff --git a/net/remote/remote-sanity.sh b/net/remote/remote-sanity.sh index 7f0bf257dc..880a480b07 100755 --- a/net/remote/remote-sanity.sh +++ b/net/remote/remote-sanity.sh @@ -3,27 +3,32 @@ cd "$(dirname "$0")"/../.. deployMethod= -leaderIp= +entrypointIp= numNodes= -# shellcheck source=/dev/null # deployConfig is written by remote_sanity.sh + +[[ -r deployConfig ]] || { + echo deployConfig missing + exit 1 +} +# shellcheck source=/dev/null # deployConfig is written by remote-node.sh source deployConfig [[ -n $deployMethod ]] || { - echo "deployMethod empty" + echo deployMethod empty exit 1 } -[[ -n $leaderIp ]] || { - echo "leaderIp empty" +[[ -n $entrypointIp ]] || { + echo entrypointIp empty exit 1 } [[ -n $numNodes ]] || { - echo "numNodes empty" + echo numNodes empty exit 1 } ledgerVerify=true validatorSanity=true -while [[ $1 = "-o" ]]; do +while [[ $1 = -o ]]; do opt="$2" shift 2 case $opt in @@ -56,7 +61,7 @@ local) PATH="$HOME"/.cargo/bin:"$PATH" export USE_INSTALL=1 - solana_bench_tps="multinode-demo/client.sh $leaderIp:~/solana" + solana_bench_tps="multinode-demo/client.sh $entrypointIp:~/solana" solana_ledger_tool=solana-ledger-tool ledger=config/ledger ;; @@ -66,19 +71,19 @@ local) esac -echo "--- $leaderIp: wallet sanity" +echo "--- $entrypointIp: wallet sanity" ( set -x - multinode-demo/test/wallet-sanity.sh "$leaderIp" + multinode-demo/test/wallet-sanity.sh "$entrypointIp" ) -echo "--- $leaderIp: node count" +echo "--- $entrypointIp: node count" ( set -x $solana_bench_tps --num-nodes "$numNodes" --converge-only ) -echo "--- $leaderIp: verify ledger" +echo "--- $entrypointIp: verify ledger" if $ledgerVerify; then if [[ -d $ledger ]]; then ( @@ -97,12 +102,12 @@ else fi -echo "--- $leaderIp: validator sanity" +echo "--- $entrypointIp: validator sanity" if $validatorSanity; then ( set -ex -o pipefail ./multinode-demo/setup.sh -t validator - timeout 10s ./multinode-demo/validator.sh "$leaderIp" "$leaderIp:8001" 2>&1 | tee validator.log + timeout 10s ./multinode-demo/validator.sh "$entrypointIp" "$entrypointIp:8001" 2>&1 | tee validator.log ) || { exitcode=$? [[ $exitcode -eq 124 ]] || exit $exitcode