diff --git a/net/net.sh b/net/net.sh index 3f5d8d37b6..1eae72b039 100755 --- a/net/net.sh +++ b/net/net.sh @@ -18,16 +18,21 @@ usage: $0 [start|stop] Operate a configured testnet start - Start the network + sanity - Sanity check the network stop - Stop the network start-specific options: - -S snapFilename - Deploy the specified Snap file - -s edge|beta|stable - Deploy the latest Snap on the specified Snap release channel - -a "setup args" - Optional additional arguments for ./multinode-demo/setup.sh + -S snapFilename - Deploy the specified Snap file + -s edge|beta|stable - Deploy the latest Snap on the specified Snap release channel + -a "setup args" - Optional additional arguments for ./multinode-demo/setup.sh Note: if RUST_LOG is set in the environment it will be propogated into the network nodes. + sanity-specific options: + -o noLedgerVerify - Skip ledger verification + -o noValidatorSanity - Skip validatory sanity + stop-specific options: none @@ -39,13 +44,15 @@ snapChannel= snapFilename= nodeSetupArgs= deployMethod=local +sanityExtraArgs= command=$1 [[ -n $command ]] || usage shift -[[ $command = start || $command = stop ]] || usage "Invalid command: $command" +[[ $command = start || $command = sanity || $command = stop ]] || + usage "Invalid command: $command" -while getopts "h?S:s:a:" opt; do +while getopts "h?S:s:a:o:" opt; do case $opt in h | \?) usage @@ -69,6 +76,17 @@ while getopts "h?S:s:a:" opt; do a) nodeSetupArgs="$OPTARG" ;; + o) + case $OPTARG in + noLedgerVerify|noValidatorSanity) + sanityExtraArgs="$sanityExtraArgs -o $OPTARG" + ;; + *) + echo "Error: unknown option: $OPTARG" + exit 1 + ;; + esac + ;; *) usage "Error: unhandled option: $opt" ;; @@ -148,7 +166,7 @@ startValidator() { set -x ssh "${sshOptions[@]}" -f "$ipAddress" \ "./solana/net/remote/remote_node.sh $deployMethod validator $leaderIp \"$nodeSetupArgs\" \"$RUST_LOG\"" - ) >> "$logFile" + ) >> "$logFile" } startClient() { @@ -160,13 +178,24 @@ startClient() { declare expectedNodeCount=$((${#validatorIpList[@]} + 1)) - ssh "${sshOptions[@]}" -f "$ipAddress" \ - "./solana/net/remote/remote_client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\"" >> "$logFile" + ( + set -x + ssh "${sshOptions[@]}" -f "$ipAddress" \ + "./solana/net/remote/remote_client.sh $deployMethod $leaderIp $expectedNodeCount \"$RUST_LOG\"" + ) >> "$logFile" +} + +sanity() { + declare expectedNodeCount=$((${#validatorIpList[@]} + 1)) + ( + set -x + # shellcheck disable=SC2029 # remote_client.sh are expanded on client side intentionally... + ssh "${sshOptions[@]}" "$leaderIp" \ + "./solana/net/remote/remote_sanity.sh $deployMethod $leaderIp $expectedNodeCount $sanityExtraArgs" + ) } start() { - [[ $command = "start" ]] || return - case $deployMethod in snap) if [[ -n $snapChannel ]]; then @@ -202,6 +231,8 @@ start() { wait validatorDeployTime=$SECONDS + sanity + SECONDS=0 for ipAddress in "${clientIpList[@]}"; do startClient "$ipAddress" "$netLogDir/client-$ipAddress.log" @@ -261,6 +292,18 @@ stop() { echo "Stopping nodes took $SECONDS seconds" } -stop -start - +case $command in +start) + stop + start + ;; +sanity) + sanity + ;; +stop) + stop + ;; +*) + echo "Internal error: Unknown command: $command" + exit 1 +esac diff --git a/net/remote/remote_sanity.sh b/net/remote/remote_sanity.sh new file mode 100755 index 0000000000..aac32bd275 --- /dev/null +++ b/net/remote/remote_sanity.sh @@ -0,0 +1,109 @@ +#!/bin/bash -e + +deployMethod="$1" +netEntrypoint="$2" +numNodes="$3" + +[[ -n $deployMethod ]] || exit +[[ -n $netEntrypoint ]] || exit +[[ -n $numNodes ]] || exit + +shift 3 + +ledgerVerify=true +validatorSanity=true +while [[ $1 = "-o" ]]; do + opt="$2" + shift 2 + case $opt in + noLedgerVerify) + ledgerVerify=false + ;; + noValidatorSanity) + validatorSanity=false + ;; + *) + echo "Error: unknown option: $opt" + exit 1 + ;; + esac +done + + +cd "$(dirname "$0")"/../.. +source net/common.sh +loadConfigFile + +case $deployMethod in +snap) + export USE_SNAP=1 + solana_bench_tps=/snap/bin/solana.bench-tps + solana_ledger_tool=/snap/bin/solana.ledger-tool + ledger=/var/snap/solana/current/config/ledger + ;; +local) + PATH="$HOME"/.cargo/bin:"$PATH" + export USE_INSTALL=1 + + solana_bench_tps=multinode-demo/client.sh + solana_ledger_tool=solana-ledger-tool + ledger=config/ledger + netEntrypoint="$:~/solana" + ;; +*) + echo "Unknown deployment method: $deployMethod" + exit 1 +esac + + +echo "--- $netEntrypoint: wallet sanity" +( + set -x + multinode-demo/test/wallet-sanity.sh "$netEntrypoint" +) + +echo "--- $netEntrypoint: node count" +( + set -x + $solana_bench_tps "$netEntrypoint" "$numNodes" -c +) + +echo "--- $netEntrypoint: verify ledger" +if $ledgerVerify; then + if [[ -d $ledger ]]; then + ( + set -x + rm -rf /var/tmp/ledger-verify + cp -r $ledger /var/tmp/ledger-verify + $solana_ledger_tool --ledger /var/tmp/ledger-verify verify + ) + else + echo "^^^ +++" + echo "Ledger verify skipped" + fi +else + echo "^^^ +++" + echo "Ledger verify skipped (NO_LEDGER_VERIFY defined)" +fi + + +echo "--- $netEntrypoint: validator sanity" +if $validatorSanity; then + ( + ./multinode-demo/setup.sh -t validator + set -e pipefail + timeout 10s ./multinode-demo/validator.sh "$netEntrypoint" 2>&1 | tee validator.log + ) + wc -l validator.log + if grep -C100 panic validator.log; then + echo "^^^ +++" + echo "Panic observed" + exit 1 + else + echo "Validator log looks ok" + fi +else + echo "^^^ +++" + echo "Validator sanity disabled (NO_VALIDATOR_SANITY defined)" +fi +