1128 lines
33 KiB
Bash
Executable File
1128 lines
33 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -e
|
|
|
|
here=$(dirname "$0")
|
|
SOLANA_ROOT="$(cd "$here"/..; pwd)"
|
|
|
|
# shellcheck source=net/common.sh
|
|
source "$here"/common.sh
|
|
|
|
usage() {
|
|
exitcode=0
|
|
if [[ -n "$1" ]]; then
|
|
exitcode=1
|
|
echo "Error: $*"
|
|
fi
|
|
CLIENT_OPTIONS=$(cat << EOM
|
|
-c clientType=numClients=extraArgs - Number of clientTypes to start. This options can be specified
|
|
more than once. Defaults to bench-tps for all clients if not
|
|
specified.
|
|
Valid client types are:
|
|
idle
|
|
bench-tps
|
|
bench-exchange
|
|
User can optionally provide extraArgs that are transparently
|
|
supplied to the client program as command line parameters.
|
|
For example,
|
|
-c bench-tps=2="--tx_count 25000"
|
|
This will start 2 bench-tps clients, and supply "--tx_count 25000"
|
|
to the bench-tps client.
|
|
EOM
|
|
)
|
|
cat <<EOF
|
|
usage: $0 [start|stop|restart|sanity] [command-specific options]
|
|
|
|
Operate a configured testnet
|
|
|
|
start - Start the network
|
|
sanity - Sanity check the network
|
|
stop - Stop the network
|
|
restart - Shortcut for stop then start
|
|
logs - Fetch remote logs from each network node
|
|
startnode - Start an individual node (previously stopped with stopNode)
|
|
stopnode - Stop an individual node
|
|
startclients - Start client nodes only
|
|
update - Deploy a new software update to the cluster
|
|
upgrade - Upgrade software on bootstrap validator. (Restart bootstrap validator manually to run it)
|
|
|
|
start-specific options:
|
|
-T [tarFilename] - Deploy the specified release tarball
|
|
-t edge|beta|stable|vX.Y.Z - Deploy the latest tarball release for the
|
|
specified release channel (edge|beta|stable) or release tag
|
|
(vX.Y.Z)
|
|
-r / --skip-setup - Reuse existing node/ledger configuration from a
|
|
previous |start| (ie, don't run ./multinode-demo/setup.sh).
|
|
-d / --debug - Build/deploy the testnet with debug binaries
|
|
$CLIENT_OPTIONS
|
|
--client-delay-start
|
|
- Number of seconds to wait after validators have finished starting before starting client programs
|
|
(default: $clientDelayStart)
|
|
-n NUM_VALIDATORS - Number of validators to apply command to.
|
|
--gpu-mode GPU_MODE - Specify GPU mode to launch validators with (default: $gpuMode).
|
|
MODE must be one of
|
|
on - GPU *required*, any vendor *
|
|
off - No GPU, CPU-only
|
|
auto - Use GPU if available, any vendor *
|
|
cuda - GPU *required*, Nvidia CUDA only
|
|
* Currently, Nvidia CUDA is the only supported GPU vendor
|
|
--hashes-per-tick NUM_HASHES|sleep|auto
|
|
- Override the default --hashes-per-tick for the cluster
|
|
--no-airdrop
|
|
- If set, disables the faucet keypair. Nodes must be funded in genesis config
|
|
--faucet-lamports NUM_LAMPORTS_TO_MINT
|
|
- Override the default 500000000000000000 lamports minted in genesis
|
|
--extra-primordial-stakes NUM_EXTRA_PRIMORDIAL_STAKES
|
|
- Number of extra nodes to be initially staked in genesis.
|
|
Implies --wait-for-supermajority 1 --async-node-init and the supermajority
|
|
wait slot may be overridden with the corresponding flag
|
|
--internal-nodes-stake-lamports NUM_LAMPORTS_PER_NODE
|
|
- Amount to stake internal nodes.
|
|
--internal-nodes-lamports NUM_LAMPORTS_PER_NODE
|
|
- Amount to fund internal nodes in genesis config.
|
|
--external-accounts-file FILE_PATH
|
|
- A YML file with a list of account pubkeys and corresponding lamport balances
|
|
in genesis config for external nodes
|
|
--no-snapshot-fetch
|
|
- If set, disables booting validators from a snapshot
|
|
--skip-poh-verify
|
|
- If set, validators will skip verifying
|
|
the ledger they already have saved to disk at
|
|
boot (results in a much faster boot)
|
|
--no-deploy
|
|
- Don't deploy new software, use the
|
|
existing deployment
|
|
--no-build
|
|
- Don't build new software, deploy the
|
|
existing binaries
|
|
|
|
--deploy-if-newer - Only deploy if newer software is
|
|
available (requires -t or -T)
|
|
|
|
--cluster-type development|devnet|testnet|mainnet-beta
|
|
- Specify whether or not to launch the cluster in "development" mode with all features enabled at epoch 0,
|
|
or various other live clusters' feature set (default: development)
|
|
--warp-slot WARP_SLOT - Boot from a snapshot that has warped ahead to WARP_SLOT rather than a slot 0 genesis.
|
|
sanity/start-specific options:
|
|
-F - Discard validator nodes that didn't bootup successfully
|
|
-o noInstallCheck - Skip solana-install sanity
|
|
-o rejectExtraNodes - Require the exact number of nodes
|
|
|
|
stop-specific options:
|
|
none
|
|
|
|
logs-specific options:
|
|
none
|
|
|
|
netem-specific options:
|
|
--config - Netem configuration (as a double quoted string)
|
|
--parition - Percentage of network that should be configured with netem
|
|
--config-file - Configuration file for partition and netem configuration
|
|
--netem-cmd - Optional command argument to netem. Default is "add". Use "cleanup" to remove rules.
|
|
|
|
update-specific options:
|
|
--platform linux|osx|windows - Deploy the tarball using 'solana-install deploy ...' for the
|
|
given platform (multiple platforms may be specified)
|
|
(-t option must be supplied as well)
|
|
|
|
startnode/stopnode-specific options:
|
|
-i [ip address] - IP Address of the node to start or stop
|
|
|
|
startclients-specific options:
|
|
$CLIENT_OPTIONS
|
|
|
|
Note: if RUST_LOG is set in the environment it will be propogated into the
|
|
network nodes.
|
|
EOF
|
|
exit $exitcode
|
|
}
|
|
|
|
initLogDir() { # Initializes the netLogDir global variable. Idempotent
|
|
[[ -z $netLogDir ]] || return 0
|
|
|
|
netLogDir="$netDir"/log
|
|
declare netLogDateDir
|
|
netLogDateDir="$netDir"/log-$(date +"%Y-%m-%d_%H_%M_%S")
|
|
if [[ -d $netLogDir && ! -L $netLogDir ]]; then
|
|
echo "Warning: moving $netLogDir to make way for symlink."
|
|
mv "$netLogDir" "$netDir"/log.old
|
|
elif [[ -L $netLogDir ]]; then
|
|
rm "$netLogDir"
|
|
fi
|
|
mkdir -p "$netConfigDir" "$netLogDateDir"
|
|
ln -sf "$netLogDateDir" "$netLogDir"
|
|
echo "Log directory: $netLogDateDir"
|
|
}
|
|
|
|
annotate() {
|
|
[[ -z $BUILDKITE ]] || {
|
|
buildkite-agent annotate "$@"
|
|
}
|
|
}
|
|
|
|
annotateBlockexplorerUrl() {
|
|
declare blockstreamer=${blockstreamerIpList[0]}
|
|
|
|
if [[ -n $blockstreamer ]]; then
|
|
annotate --style info --context blockexplorer-url "Block explorer: http://$blockstreamer/"
|
|
fi
|
|
}
|
|
|
|
build() {
|
|
supported=("18.04")
|
|
declare MAYBE_DOCKER=
|
|
if [[ $(uname) != Linux || ! " ${supported[*]} " =~ $(lsb_release -sr) ]]; then
|
|
# shellcheck source=ci/rust-version.sh
|
|
source "$SOLANA_ROOT"/ci/rust-version.sh
|
|
MAYBE_DOCKER="ci/docker-run.sh $rust_stable_docker_image"
|
|
fi
|
|
SECONDS=0
|
|
(
|
|
cd "$SOLANA_ROOT"
|
|
echo "--- Build started at $(date)"
|
|
|
|
set -x
|
|
rm -rf farf
|
|
|
|
buildVariant=
|
|
if $debugBuild; then
|
|
buildVariant=debug
|
|
fi
|
|
|
|
$MAYBE_DOCKER bash -c "
|
|
set -ex
|
|
scripts/cargo-install-all.sh farf \"$buildVariant\"
|
|
"
|
|
)
|
|
echo "Build took $SECONDS seconds"
|
|
}
|
|
|
|
startCommon() {
|
|
declare ipAddress=$1
|
|
test -d "$SOLANA_ROOT"
|
|
if $skipSetup; then
|
|
ssh "${sshOptions[@]}" "$ipAddress" "
|
|
set -x;
|
|
mkdir -p ~/solana/config;
|
|
rm -rf ~/config;
|
|
mv ~/solana/config ~;
|
|
rm -rf ~/solana;
|
|
mkdir -p ~/solana ~/.cargo/bin;
|
|
mv ~/config ~/solana/
|
|
"
|
|
else
|
|
ssh "${sshOptions[@]}" "$ipAddress" "
|
|
set -x;
|
|
rm -rf ~/solana;
|
|
mkdir -p ~/.cargo/bin
|
|
"
|
|
fi
|
|
[[ -z "$externalNodeSshKey" ]] || ssh-copy-id -f -i "$externalNodeSshKey" "${sshOptions[@]}" "solana@$ipAddress"
|
|
syncScripts "$ipAddress"
|
|
}
|
|
|
|
syncScripts() {
|
|
echo "rsyncing scripts... to $ipAddress"
|
|
declare ipAddress=$1
|
|
rsync -vPrc -e "ssh ${sshOptions[*]}" \
|
|
--exclude 'net/log*' \
|
|
"$SOLANA_ROOT"/{fetch-perf-libs.sh,fetch-spl.sh,scripts,net,multinode-demo} \
|
|
"$ipAddress":~/solana/ > /dev/null
|
|
}
|
|
|
|
# Deploy local binaries to bootstrap validator. Other validators and clients later fetch the
|
|
# binaries from it
|
|
deployBootstrapValidator() {
|
|
declare ipAddress=$1
|
|
|
|
echo "Deploying software to bootstrap validator ($ipAddress)"
|
|
case $deployMethod in
|
|
tar)
|
|
rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/bin/* "$ipAddress:~/.cargo/bin/"
|
|
rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/version.yml "$ipAddress:~/"
|
|
;;
|
|
local)
|
|
rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/"
|
|
ssh "${sshOptions[@]}" -n "$ipAddress" "rm -f ~/version.yml; touch ~/version.yml"
|
|
;;
|
|
skip)
|
|
;;
|
|
*)
|
|
usage "Internal error: invalid deployMethod: $deployMethod"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
startBootstrapLeader() {
|
|
declare ipAddress=$1
|
|
declare nodeIndex="$2"
|
|
declare logFile="$3"
|
|
echo "--- Starting bootstrap validator: $ipAddress"
|
|
echo "start log: $logFile"
|
|
|
|
(
|
|
set -x
|
|
startCommon "$ipAddress" || exit 1
|
|
[[ -z "$externalPrimordialAccountsFile" ]] || rsync -vPrc -e "ssh ${sshOptions[*]}" "$externalPrimordialAccountsFile" \
|
|
"$ipAddress:$remoteExternalPrimordialAccountsFile"
|
|
|
|
deployBootstrapValidator "$ipAddress"
|
|
|
|
ssh "${sshOptions[@]}" -n "$ipAddress" \
|
|
"./solana/net/remote/remote-node.sh \
|
|
$deployMethod \
|
|
bootstrap-validator \
|
|
$entrypointIp \
|
|
$((${#validatorIpList[@]} + ${#blockstreamerIpList[@]})) \
|
|
\"$RUST_LOG\" \
|
|
$skipSetup \
|
|
$failOnValidatorBootupFailure \
|
|
\"$remoteExternalPrimordialAccountsFile\" \
|
|
\"$maybeDisableAirdrops\" \
|
|
\"$internalNodesStakeLamports\" \
|
|
\"$internalNodesLamports\" \
|
|
$nodeIndex \
|
|
${#clientIpList[@]} \"$benchTpsExtraArgs\" \
|
|
${#clientIpList[@]} \"$benchExchangeExtraArgs\" \
|
|
\"$genesisOptions\" \
|
|
\"$maybeNoSnapshot $maybeSkipLedgerVerify $maybeLimitLedgerSize $maybeWaitForSupermajority\" \
|
|
\"$gpuMode\" \
|
|
\"$maybeWarpSlot\" \
|
|
\"$waitForNodeInit\" \
|
|
\"$extraPrimordialStakes\" \
|
|
"
|
|
|
|
) >> "$logFile" 2>&1 || {
|
|
cat "$logFile"
|
|
echo "^^^ +++"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
startNode() {
|
|
declare ipAddress=$1
|
|
declare nodeType=$2
|
|
declare nodeIndex="$3"
|
|
|
|
initLogDir
|
|
declare logFile="$netLogDir/validator-$ipAddress.log"
|
|
|
|
if [[ -z $nodeType ]]; then
|
|
echo nodeType not specified
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z $nodeIndex ]]; then
|
|
echo nodeIndex not specified
|
|
exit 1
|
|
fi
|
|
|
|
echo "--- Starting $nodeType: $ipAddress"
|
|
echo "start log: $logFile"
|
|
(
|
|
set -x
|
|
startCommon "$ipAddress"
|
|
|
|
if [[ $nodeType = blockstreamer ]] && [[ -n $letsEncryptDomainName ]]; then
|
|
#
|
|
# Create/renew TLS certificate
|
|
#
|
|
declare localArchive=~/letsencrypt-"$letsEncryptDomainName".tgz
|
|
if [[ -r "$localArchive" ]]; then
|
|
timeout 30s scp "${sshOptions[@]}" "$localArchive" "$ipAddress:letsencrypt.tgz"
|
|
fi
|
|
ssh "${sshOptions[@]}" -n "$ipAddress" \
|
|
"sudo -H /certbot-restore.sh $letsEncryptDomainName maintainers@solana.foundation"
|
|
rm -f letsencrypt.tgz
|
|
timeout 30s scp "${sshOptions[@]}" "$ipAddress:/letsencrypt.tgz" letsencrypt.tgz
|
|
test -s letsencrypt.tgz # Ensure non-empty before overwriting $localArchive
|
|
cp letsencrypt.tgz "$localArchive"
|
|
fi
|
|
|
|
ssh "${sshOptions[@]}" -n "$ipAddress" \
|
|
"./solana/net/remote/remote-node.sh \
|
|
$deployMethod \
|
|
$nodeType \
|
|
$entrypointIp \
|
|
$((${#validatorIpList[@]} + ${#blockstreamerIpList[@]})) \
|
|
\"$RUST_LOG\" \
|
|
$skipSetup \
|
|
$failOnValidatorBootupFailure \
|
|
\"$remoteExternalPrimordialAccountsFile\" \
|
|
\"$maybeDisableAirdrops\" \
|
|
\"$internalNodesStakeLamports\" \
|
|
\"$internalNodesLamports\" \
|
|
$nodeIndex \
|
|
${#clientIpList[@]} \"$benchTpsExtraArgs\" \
|
|
${#clientIpList[@]} \"$benchExchangeExtraArgs\" \
|
|
\"$genesisOptions\" \
|
|
\"$maybeNoSnapshot $maybeSkipLedgerVerify $maybeLimitLedgerSize $maybeWaitForSupermajority\" \
|
|
\"$gpuMode\" \
|
|
\"$maybeWarpSlot\" \
|
|
\"$waitForNodeInit\" \
|
|
\"$extraPrimordialStakes\" \
|
|
"
|
|
) >> "$logFile" 2>&1 &
|
|
declare pid=$!
|
|
ln -sf "validator-$ipAddress.log" "$netLogDir/validator-$pid.log"
|
|
pids+=("$pid")
|
|
}
|
|
|
|
startClient() {
|
|
declare ipAddress=$1
|
|
declare clientToRun="$2"
|
|
declare clientIndex="$3"
|
|
|
|
initLogDir
|
|
declare logFile="$netLogDir/client-$clientToRun-$ipAddress.log"
|
|
|
|
echo "--- Starting client: $ipAddress - $clientToRun"
|
|
echo "start log: $logFile"
|
|
(
|
|
set -x
|
|
startCommon "$ipAddress"
|
|
ssh "${sshOptions[@]}" -f "$ipAddress" \
|
|
"./solana/net/remote/remote-client.sh $deployMethod $entrypointIp \
|
|
$clientToRun \"$RUST_LOG\" \"$benchTpsExtraArgs\" \"$benchExchangeExtraArgs\" $clientIndex"
|
|
) >> "$logFile" 2>&1 || {
|
|
cat "$logFile"
|
|
echo "^^^ +++"
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
startClients() {
|
|
for ((i=0; i < "$numClients" && i < "$numClientsRequested"; i++)) do
|
|
if [[ $i -lt "$numBenchTpsClients" ]]; then
|
|
startClient "${clientIpList[$i]}" "solana-bench-tps" "$i"
|
|
elif [[ $i -lt $((numBenchTpsClients + numBenchExchangeClients)) ]]; then
|
|
startClient "${clientIpList[$i]}" "solana-bench-exchange" $((i-numBenchTpsClients))
|
|
else
|
|
startClient "${clientIpList[$i]}" "idle"
|
|
fi
|
|
done
|
|
}
|
|
|
|
sanity() {
|
|
declare skipBlockstreamerSanity=$1
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-sanity-begin=1"
|
|
|
|
declare ok=true
|
|
declare bootstrapLeader=${validatorIpList[0]}
|
|
declare blockstreamer=${blockstreamerIpList[0]}
|
|
|
|
annotateBlockexplorerUrl
|
|
|
|
echo "--- Sanity: $bootstrapLeader"
|
|
(
|
|
set -x
|
|
# shellcheck disable=SC2029 # remote-client.sh args are expanded on client side intentionally
|
|
ssh "${sshOptions[@]}" "$bootstrapLeader" \
|
|
"./solana/net/remote/remote-sanity.sh $bootstrapLeader $sanityExtraArgs \"$RUST_LOG\""
|
|
) || ok=false
|
|
$ok || exit 1
|
|
|
|
if [[ -z $skipBlockstreamerSanity && -n $blockstreamer ]]; then
|
|
# If there's a blockstreamer node run a reduced sanity check on it as well
|
|
echo "--- Sanity: $blockstreamer"
|
|
(
|
|
set -x
|
|
# shellcheck disable=SC2029 # remote-client.sh args are expanded on client side intentionally
|
|
ssh "${sshOptions[@]}" "$blockstreamer" \
|
|
"./solana/net/remote/remote-sanity.sh $blockstreamer $sanityExtraArgs \"$RUST_LOG\""
|
|
) || ok=false
|
|
$ok || exit 1
|
|
fi
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-sanity-complete=1"
|
|
}
|
|
|
|
deployUpdate() {
|
|
if [[ -z $updatePlatforms ]]; then
|
|
echo "No update platforms"
|
|
return
|
|
fi
|
|
if [[ -z $releaseChannel ]]; then
|
|
echo "Release channel not specified (use -t option)"
|
|
exit 1
|
|
fi
|
|
|
|
declare ok=true
|
|
declare bootstrapLeader=${validatorIpList[0]}
|
|
|
|
for updatePlatform in $updatePlatforms; do
|
|
echo "--- Deploying solana-install update: $updatePlatform"
|
|
(
|
|
set -x
|
|
|
|
scripts/solana-install-update-manifest-keypair.sh "$updatePlatform"
|
|
|
|
timeout 30s scp "${sshOptions[@]}" \
|
|
update_manifest_keypair.json "$bootstrapLeader:solana/update_manifest_keypair.json"
|
|
|
|
# shellcheck disable=SC2029 # remote-deploy-update.sh args are expanded on client side intentionally
|
|
ssh "${sshOptions[@]}" "$bootstrapLeader" \
|
|
"./solana/net/remote/remote-deploy-update.sh $releaseChannel $updatePlatform"
|
|
) || ok=false
|
|
$ok || exit 1
|
|
done
|
|
}
|
|
|
|
getNodeType() {
|
|
echo "getNodeType: $nodeAddress"
|
|
[[ -n $nodeAddress ]] || {
|
|
echo "Error: nodeAddress not set"
|
|
exit 1
|
|
}
|
|
nodeIndex=0 # <-- global
|
|
nodeType=validator # <-- global
|
|
|
|
for ipAddress in "${validatorIpList[@]}" b "${blockstreamerIpList[@]}"; do
|
|
if [[ $ipAddress = b ]]; then
|
|
nodeType=blockstreamer
|
|
continue
|
|
fi
|
|
|
|
if [[ $ipAddress = "$nodeAddress" ]]; then
|
|
echo "getNodeType: $nodeType ($nodeIndex)"
|
|
return
|
|
fi
|
|
((nodeIndex = nodeIndex + 1))
|
|
done
|
|
|
|
echo "Error: Unknown node: $nodeAddress"
|
|
exit 1
|
|
}
|
|
|
|
prepareDeploy() {
|
|
case $deployMethod in
|
|
tar)
|
|
if [[ -n $releaseChannel ]]; then
|
|
rm -f "$SOLANA_ROOT"/solana-release.tar.bz2
|
|
declare updateDownloadUrl=http://release.solana.com/"$releaseChannel"/solana-release-x86_64-unknown-linux-gnu.tar.bz2
|
|
(
|
|
set -x
|
|
curl --retry 5 --retry-delay 2 --retry-connrefused \
|
|
-o "$SOLANA_ROOT"/solana-release.tar.bz2 "$updateDownloadUrl"
|
|
)
|
|
tarballFilename="$SOLANA_ROOT"/solana-release.tar.bz2
|
|
fi
|
|
(
|
|
set -x
|
|
rm -rf "$SOLANA_ROOT"/solana-release
|
|
(cd "$SOLANA_ROOT"; tar jxv) < "$tarballFilename"
|
|
cat "$SOLANA_ROOT"/solana-release/version.yml
|
|
)
|
|
;;
|
|
local)
|
|
if $doBuild; then
|
|
build
|
|
else
|
|
echo "Build skipped due to --no-build"
|
|
fi
|
|
;;
|
|
skip)
|
|
;;
|
|
*)
|
|
usage "Internal error: invalid deployMethod: $deployMethod"
|
|
;;
|
|
esac
|
|
|
|
if [[ -n $deployIfNewer ]]; then
|
|
if [[ $deployMethod != tar ]]; then
|
|
echo "Error: --deploy-if-newer only supported for tar deployments"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Fetching current software version"
|
|
(
|
|
set -x
|
|
rsync -vPrc -e "ssh ${sshOptions[*]}" "${validatorIpList[0]}":~/version.yml current-version.yml
|
|
)
|
|
cat current-version.yml
|
|
if ! diff -q current-version.yml "$SOLANA_ROOT"/solana-release/version.yml; then
|
|
echo "Cluster software version is old. Update required"
|
|
else
|
|
echo "Cluster software version is current. No update required"
|
|
exit 0
|
|
fi
|
|
fi
|
|
}
|
|
|
|
deploy() {
|
|
initLogDir
|
|
|
|
echo "Deployment started at $(date)"
|
|
$metricsWriteDatapoint "testnet-deploy net-start-begin=1"
|
|
|
|
declare bootstrapLeader=true
|
|
for nodeAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}"; do
|
|
nodeType=
|
|
nodeIndex=
|
|
getNodeType
|
|
if $bootstrapLeader; then
|
|
SECONDS=0
|
|
declare bootstrapNodeDeployTime=
|
|
startBootstrapLeader "$nodeAddress" $nodeIndex "$netLogDir/bootstrap-validator-$ipAddress.log"
|
|
bootstrapNodeDeployTime=$SECONDS
|
|
$metricsWriteDatapoint "testnet-deploy net-bootnode-leader-started=1"
|
|
|
|
bootstrapLeader=false
|
|
SECONDS=0
|
|
pids=()
|
|
else
|
|
startNode "$ipAddress" $nodeType $nodeIndex
|
|
|
|
# Stagger additional node start time. If too many nodes start simultaneously
|
|
# the bootstrap node gets more rsync requests from the additional nodes than
|
|
# it can handle.
|
|
sleep 2
|
|
fi
|
|
done
|
|
|
|
|
|
for pid in "${pids[@]}"; do
|
|
declare ok=true
|
|
wait "$pid" || ok=false
|
|
if ! $ok; then
|
|
echo "+++ validator failed to start"
|
|
cat "$netLogDir/validator-$pid.log"
|
|
if $failOnValidatorBootupFailure; then
|
|
exit 1
|
|
else
|
|
echo "Failure is non-fatal"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if ! $waitForNodeInit; then
|
|
# Handle async init
|
|
declare startTime=$SECONDS
|
|
for ipAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}"; do
|
|
declare timeWaited=$((SECONDS - startTime))
|
|
if [[ $timeWaited -gt 600 ]]; then
|
|
break
|
|
fi
|
|
ssh "${sshOptions[@]}" -n "$ipAddress" \
|
|
"./solana/net/remote/remote-node-wait-init.sh $((600 - timeWaited))"
|
|
done
|
|
fi
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-validators-started=1"
|
|
additionalNodeDeployTime=$SECONDS
|
|
|
|
annotateBlockexplorerUrl
|
|
|
|
sanity skipBlockstreamerSanity # skip sanity on blockstreamer node, it may not
|
|
# have caught up to the bootstrap validator yet
|
|
|
|
echo "--- Sleeping $clientDelayStart seconds after validators are started before starting clients"
|
|
sleep "$clientDelayStart"
|
|
|
|
SECONDS=0
|
|
startClients
|
|
clientDeployTime=$SECONDS
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-start-complete=1"
|
|
|
|
declare networkVersion=unknown
|
|
case $deployMethod in
|
|
tar)
|
|
networkVersion="$(
|
|
(
|
|
set -o pipefail
|
|
grep "^commit: " "$SOLANA_ROOT"/solana-release/version.yml | head -n1 | cut -d\ -f2
|
|
) || echo "tar-unknown"
|
|
)"
|
|
;;
|
|
local)
|
|
networkVersion="$(git rev-parse HEAD || echo local-unknown)"
|
|
;;
|
|
skip)
|
|
;;
|
|
*)
|
|
usage "Internal error: invalid deployMethod: $deployMethod"
|
|
;;
|
|
esac
|
|
$metricsWriteDatapoint "testnet-deploy version=\"${networkVersion:0:9}\""
|
|
|
|
echo
|
|
echo "--- Deployment Successful"
|
|
echo "Bootstrap validator deployment took $bootstrapNodeDeployTime seconds"
|
|
echo "Additional validator deployment (${#validatorIpList[@]} validators, ${#blockstreamerIpList[@]} blockstreamer nodes) took $additionalNodeDeployTime seconds"
|
|
echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds"
|
|
echo "Network start logs in $netLogDir"
|
|
}
|
|
|
|
stopNode() {
|
|
local ipAddress=$1
|
|
local block=$2
|
|
|
|
initLogDir
|
|
declare logFile="$netLogDir/stop-validator-$ipAddress.log"
|
|
|
|
echo "--- Stopping node: $ipAddress"
|
|
echo "stop log: $logFile"
|
|
syncScripts "$ipAddress"
|
|
(
|
|
# Since cleanup.sh does a pkill, we cannot pass the command directly,
|
|
# otherwise the process which is doing the killing will be killed because
|
|
# the script itself will match the pkill pattern
|
|
set -x
|
|
# shellcheck disable=SC2029 # It's desired that PS4 be expanded on the client side
|
|
ssh "${sshOptions[@]}" "$ipAddress" "PS4=\"$PS4\" ./solana/net/remote/cleanup.sh"
|
|
) >> "$logFile" 2>&1 &
|
|
|
|
declare pid=$!
|
|
ln -sf "stop-validator-$ipAddress.log" "$netLogDir/stop-validator-$pid.log"
|
|
if $block; then
|
|
wait $pid
|
|
else
|
|
pids+=("$pid")
|
|
fi
|
|
}
|
|
|
|
stop() {
|
|
SECONDS=0
|
|
$metricsWriteDatapoint "testnet-deploy net-stop-begin=1"
|
|
|
|
declare loopCount=0
|
|
pids=()
|
|
for ipAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}" "${clientIpList[@]}"; do
|
|
stopNode "$ipAddress" false
|
|
|
|
# Stagger additional node stop time to avoid too many concurrent ssh
|
|
# sessions
|
|
((loopCount++ % 4 == 0)) && sleep 2
|
|
done
|
|
|
|
echo --- Waiting for nodes to finish stopping
|
|
for pid in "${pids[@]}"; do
|
|
echo -n "$pid "
|
|
wait "$pid" || true
|
|
done
|
|
echo
|
|
|
|
$metricsWriteDatapoint "testnet-deploy net-stop-complete=1"
|
|
echo "Stopping nodes took $SECONDS seconds"
|
|
}
|
|
|
|
checkPremptibleInstances() {
|
|
# The validatorIpList nodes may be preemptible instances that can disappear at
|
|
# any time. Try to detect when a validator has been preempted to help the user
|
|
# out.
|
|
#
|
|
# Of course this isn't airtight as an instance could always disappear
|
|
# immediately after its successfully pinged.
|
|
for ipAddress in "${validatorIpList[@]}"; do
|
|
(
|
|
set -x
|
|
timeout 5s ping -c 1 "$ipAddress" | tr - _
|
|
) || {
|
|
cat <<EOF
|
|
|
|
Warning: $ipAddress may have been preempted.
|
|
|
|
Run |./gce.sh config| to restart it
|
|
EOF
|
|
exit 1
|
|
}
|
|
done
|
|
}
|
|
|
|
releaseChannel=
|
|
deployMethod=local
|
|
deployIfNewer=
|
|
sanityExtraArgs=
|
|
skipSetup=false
|
|
updatePlatforms=
|
|
nodeAddress=
|
|
numIdleClients=0
|
|
numBenchTpsClients=0
|
|
numBenchExchangeClients=0
|
|
benchTpsExtraArgs=
|
|
benchExchangeExtraArgs=
|
|
failOnValidatorBootupFailure=true
|
|
genesisOptions=
|
|
numValidatorsRequested=
|
|
externalPrimordialAccountsFile=
|
|
remoteExternalPrimordialAccountsFile=
|
|
internalNodesStakeLamports=
|
|
internalNodesLamports=
|
|
maybeNoSnapshot=""
|
|
maybeLimitLedgerSize=""
|
|
maybeSkipLedgerVerify=""
|
|
maybeDisableAirdrops=""
|
|
maybeWaitForSupermajority=""
|
|
debugBuild=false
|
|
doBuild=true
|
|
gpuMode=auto
|
|
netemPartition=""
|
|
netemConfig=""
|
|
netemConfigFile=""
|
|
netemCommand="add"
|
|
clientDelayStart=0
|
|
netLogDir=
|
|
maybeWarpSlot=
|
|
waitForNodeInit=true
|
|
extraPrimordialStakes=0
|
|
|
|
command=$1
|
|
[[ -n $command ]] || usage
|
|
shift
|
|
|
|
shortArgs=()
|
|
while [[ -n $1 ]]; do
|
|
if [[ ${1:0:2} = -- ]]; then
|
|
if [[ $1 = --hashes-per-tick ]]; then
|
|
genesisOptions="$genesisOptions $1 $2"
|
|
shift 2
|
|
elif [[ $1 = --slots-per-epoch ]]; then
|
|
genesisOptions="$genesisOptions $1 $2"
|
|
shift 2
|
|
elif [[ $1 = --target-lamports-per-signature ]]; then
|
|
genesisOptions="$genesisOptions $1 $2"
|
|
shift 2
|
|
elif [[ $1 = --faucet-lamports ]]; then
|
|
genesisOptions="$genesisOptions $1 $2"
|
|
shift 2
|
|
elif [[ $1 = --cluster-type ]]; then
|
|
case "$2" in
|
|
development|devnet|testnet|mainnet-beta)
|
|
;;
|
|
*)
|
|
echo "Unexpected cluster type: \"$2\""
|
|
exit 1
|
|
;;
|
|
esac
|
|
genesisOptions="$genesisOptions $1 $2"
|
|
shift 2
|
|
elif [[ $1 = --no-snapshot-fetch ]]; then
|
|
maybeNoSnapshot="$1"
|
|
shift 1
|
|
elif [[ $1 = --deploy-if-newer ]]; then
|
|
deployIfNewer=1
|
|
shift 1
|
|
elif [[ $1 = --no-deploy ]]; then
|
|
deployMethod=skip
|
|
shift 1
|
|
elif [[ $1 = --no-build ]]; then
|
|
doBuild=false
|
|
shift 1
|
|
elif [[ $1 = --limit-ledger-size ]]; then
|
|
maybeLimitLedgerSize="$1"
|
|
shift 1
|
|
elif [[ $1 = --skip-poh-verify ]]; then
|
|
maybeSkipLedgerVerify="$1"
|
|
shift 1
|
|
elif [[ $1 = --skip-setup ]]; then
|
|
skipSetup=true
|
|
shift 1
|
|
elif [[ $1 = --platform ]]; then
|
|
updatePlatforms="$updatePlatforms $2"
|
|
shift 2
|
|
elif [[ $1 = --internal-nodes-stake-lamports ]]; then
|
|
internalNodesStakeLamports="$2"
|
|
shift 2
|
|
elif [[ $1 = --internal-nodes-lamports ]]; then
|
|
internalNodesLamports="$2"
|
|
shift 2
|
|
elif [[ $1 = --external-accounts-file ]]; then
|
|
externalPrimordialAccountsFile="$2"
|
|
remoteExternalPrimordialAccountsFile=/tmp/external-primordial-accounts.yml
|
|
shift 2
|
|
elif [[ $1 = --no-airdrop ]]; then
|
|
maybeDisableAirdrops="$1"
|
|
shift 1
|
|
elif [[ $1 = --debug ]]; then
|
|
debugBuild=true
|
|
shift 1
|
|
elif [[ $1 = --partition ]]; then
|
|
netemPartition=$2
|
|
shift 2
|
|
elif [[ $1 = --config ]]; then
|
|
netemConfig=$2
|
|
shift 2
|
|
elif [[ $1 == --config-file ]]; then
|
|
netemConfigFile=$2
|
|
shift 2
|
|
elif [[ $1 == --netem-cmd ]]; then
|
|
netemCommand=$2
|
|
shift 2
|
|
elif [[ $1 = --gpu-mode ]]; then
|
|
gpuMode=$2
|
|
case "$gpuMode" in
|
|
on|off|auto|cuda)
|
|
;;
|
|
*)
|
|
echo "Unexpected GPU mode: \"$gpuMode\""
|
|
exit 1
|
|
;;
|
|
esac
|
|
shift 2
|
|
elif [[ $1 == --client-delay-start ]]; then
|
|
clientDelayStart=$2
|
|
shift 2
|
|
elif [[ $1 == --wait-for-supermajority ]]; then
|
|
maybeWaitForSupermajority="$1 $2"
|
|
shift 2
|
|
elif [[ $1 == --warp-slot ]]; then
|
|
maybeWarpSlot="$1 $2"
|
|
shift 2
|
|
elif [[ $1 == --async-node-init ]]; then
|
|
waitForNodeInit=false
|
|
shift 1
|
|
elif [[ $1 == --extra-primordial-stakes ]]; then
|
|
extraPrimordialStakes=$2
|
|
shift 2
|
|
else
|
|
usage "Unknown long option: $1"
|
|
fi
|
|
else
|
|
shortArgs+=("$1")
|
|
shift
|
|
fi
|
|
done
|
|
|
|
while getopts "h?T:t:o:f:rc:Fn:i:d" opt "${shortArgs[@]}"; do
|
|
case $opt in
|
|
h | \?)
|
|
usage
|
|
;;
|
|
T)
|
|
tarballFilename=$OPTARG
|
|
[[ -r $tarballFilename ]] || usage "File not readable: $tarballFilename"
|
|
deployMethod=tar
|
|
;;
|
|
t)
|
|
case $OPTARG in
|
|
edge|beta|stable|v*)
|
|
releaseChannel=$OPTARG
|
|
deployMethod=tar
|
|
;;
|
|
*)
|
|
usage "Invalid release channel: $OPTARG"
|
|
;;
|
|
esac
|
|
;;
|
|
n)
|
|
numValidatorsRequested=$OPTARG
|
|
;;
|
|
r)
|
|
skipSetup=true
|
|
;;
|
|
o)
|
|
case $OPTARG in
|
|
rejectExtraNodes|noInstallCheck)
|
|
sanityExtraArgs="$sanityExtraArgs -o $OPTARG"
|
|
;;
|
|
*)
|
|
usage "Unknown option: $OPTARG"
|
|
;;
|
|
esac
|
|
;;
|
|
c)
|
|
getClientTypeAndNum() {
|
|
if ! [[ $OPTARG == *'='* ]]; then
|
|
echo "Error: Expecting tuple \"clientType=numClientType=extraArgs\" but got \"$OPTARG\""
|
|
exit 1
|
|
fi
|
|
local keyValue
|
|
IFS='=' read -ra keyValue <<< "$OPTARG"
|
|
local clientType=${keyValue[0]}
|
|
local numClients=${keyValue[1]}
|
|
local extraArgs=${keyValue[2]}
|
|
re='^[0-9]+$'
|
|
if ! [[ $numClients =~ $re ]] ; then
|
|
echo "error: numClientType must be a number but got \"$numClients\""
|
|
exit 1
|
|
fi
|
|
case $clientType in
|
|
idle)
|
|
numIdleClients=$numClients
|
|
# $extraArgs ignored for 'idle'
|
|
;;
|
|
bench-tps)
|
|
numBenchTpsClients=$numClients
|
|
benchTpsExtraArgs=$extraArgs
|
|
;;
|
|
bench-exchange)
|
|
numBenchExchangeClients=$numClients
|
|
benchExchangeExtraArgs=$extraArgs
|
|
;;
|
|
*)
|
|
echo "Unknown client type: $clientType"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
getClientTypeAndNum
|
|
;;
|
|
F)
|
|
failOnValidatorBootupFailure=false
|
|
;;
|
|
i)
|
|
nodeAddress=$OPTARG
|
|
;;
|
|
d)
|
|
debugBuild=true
|
|
;;
|
|
*)
|
|
usage "Error: unhandled option: $opt"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
loadConfigFile
|
|
|
|
if [[ -n $numValidatorsRequested ]]; then
|
|
truncatedNodeList=( "${validatorIpList[@]:0:$numValidatorsRequested}" )
|
|
unset validatorIpList
|
|
validatorIpList=( "${truncatedNodeList[@]}" )
|
|
fi
|
|
|
|
numClients=${#clientIpList[@]}
|
|
numClientsRequested=$((numBenchTpsClients + numBenchExchangeClients + numIdleClients))
|
|
if [[ "$numClientsRequested" -eq 0 ]]; then
|
|
numBenchTpsClients=$numClients
|
|
numClientsRequested=$numClients
|
|
else
|
|
if [[ "$numClientsRequested" -gt "$numClients" ]]; then
|
|
echo "Error: More clients requested ($numClientsRequested) then available ($numClients)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$maybeWaitForSupermajority" && -n "$maybeWarpSlot" ]]; then
|
|
read -r _ waitSlot <<<"$maybeWaitForSupermajority"
|
|
read -r _ warpSlot <<<"$maybeWarpSlot"
|
|
if [[ $waitSlot -ne $warpSlot ]]; then
|
|
echo "Error: When specifying both --wait-for-supermajority and --warp-slot,"
|
|
echo "they must use the same slot. ($waitSlot != $warpSlot)"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [[ $extraPrimordialStakes -gt 0 ]]; then
|
|
# Extra primoridial stakes require that all of the validators start at
|
|
# the same time. Force async init and wait for supermajority here.
|
|
waitForNodeInit=false
|
|
if [[ -z "$maybeWaitForSupermajority" ]]; then
|
|
waitSlot=
|
|
if [[ -n "$maybeWarpSlot" ]]; then
|
|
read -r _ waitSlot <<<"$maybeWarpSlot"
|
|
else
|
|
waitSlot=1
|
|
fi
|
|
maybeWaitForSupermajority="--wait-for-supermajority $waitSlot"
|
|
fi
|
|
fi
|
|
|
|
checkPremptibleInstances
|
|
|
|
case $command in
|
|
restart)
|
|
prepareDeploy
|
|
stop
|
|
deploy
|
|
;;
|
|
start)
|
|
prepareDeploy
|
|
deploy
|
|
;;
|
|
sanity)
|
|
sanity
|
|
;;
|
|
stop)
|
|
stop
|
|
;;
|
|
update)
|
|
deployUpdate
|
|
;;
|
|
upgrade)
|
|
bootstrapValidatorIp="${validatorIpList[0]}"
|
|
prepareDeploy
|
|
deployBootstrapValidator "$bootstrapValidatorIp"
|
|
# (start|stop)Node need refactored to support restarting the bootstrap validator
|
|
;;
|
|
stopnode)
|
|
if [[ -z $nodeAddress ]]; then
|
|
usage "node address (-i) not specified"
|
|
exit 1
|
|
fi
|
|
stopNode "$nodeAddress" true
|
|
;;
|
|
startnode)
|
|
if [[ -z $nodeAddress ]]; then
|
|
usage "node address (-i) not specified"
|
|
exit 1
|
|
fi
|
|
nodeType=
|
|
nodeIndex=
|
|
getNodeType
|
|
startNode "$nodeAddress" $nodeType $nodeIndex
|
|
;;
|
|
startclients)
|
|
startClients
|
|
;;
|
|
logs)
|
|
initLogDir
|
|
fetchRemoteLog() {
|
|
declare ipAddress=$1
|
|
declare log=$2
|
|
echo "--- fetching $log from $ipAddress"
|
|
(
|
|
set -x
|
|
timeout 30s scp "${sshOptions[@]}" \
|
|
"$ipAddress":solana/"$log".log "$netLogDir"/remote-"$log"-"$ipAddress".log
|
|
) || echo "failed to fetch log"
|
|
}
|
|
fetchRemoteLog "${validatorIpList[0]}" faucet
|
|
for ipAddress in "${validatorIpList[@]}"; do
|
|
fetchRemoteLog "$ipAddress" validator
|
|
done
|
|
for ipAddress in "${clientIpList[@]}"; do
|
|
fetchRemoteLog "$ipAddress" client
|
|
done
|
|
for ipAddress in "${blockstreamerIpList[@]}"; do
|
|
fetchRemoteLog "$ipAddress" validator
|
|
done
|
|
;;
|
|
netem)
|
|
if [[ -n $netemConfigFile ]]; then
|
|
remoteNetemConfigFile="$(basename "$netemConfigFile")"
|
|
if [[ $netemCommand = "add" ]]; then
|
|
for ipAddress in "${validatorIpList[@]}"; do
|
|
"$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana
|
|
done
|
|
fi
|
|
for i in "${!validatorIpList[@]}"; do
|
|
"$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \
|
|
"$netemCommand" ~solana/solana/"$remoteNetemConfigFile" "${#validatorIpList[@]}" "$i"
|
|
done
|
|
else
|
|
num_nodes=$((${#validatorIpList[@]}*netemPartition/100))
|
|
if [[ $((${#validatorIpList[@]}*netemPartition%100)) -gt 0 ]]; then
|
|
num_nodes=$((num_nodes+1))
|
|
fi
|
|
if [[ "$num_nodes" -gt "${#validatorIpList[@]}" ]]; then
|
|
num_nodes=${#validatorIpList[@]}
|
|
fi
|
|
|
|
# Stop netem on all nodes
|
|
for ipAddress in "${validatorIpList[@]}"; do
|
|
"$here"/ssh.sh solana@"$ipAddress" 'solana/scripts/netem.sh delete < solana/netem.cfg || true'
|
|
done
|
|
|
|
# Start netem on required nodes
|
|
for ((i=0; i<num_nodes; i++ )); do :
|
|
"$here"/ssh.sh solana@"${validatorIpList[$i]}" "echo $netemConfig > solana/netem.cfg; solana/scripts/netem.sh add \"$netemConfig\""
|
|
done
|
|
fi
|
|
;;
|
|
*)
|
|
echo "Internal error: Unknown command: $command"
|
|
usage
|
|
exit 1
|
|
esac
|