diff --git a/multinode-demo/setup.sh b/multinode-demo/setup.sh index 0f28bb6a8f..2a5d551565 100755 --- a/multinode-demo/setup.sh +++ b/multinode-demo/setup.sh @@ -6,12 +6,13 @@ source "$here"/common.sh setup_secondary_mount set -e -"$here"/clear-config.sh + +rm -rf "$SOLANA_CONFIG_DIR"/bootstrap-leader +mkdir -p "$SOLANA_CONFIG_DIR"/bootstrap-leader # Create genesis ledger -$solana_keygen new -o "$SOLANA_CONFIG_DIR"/mint-keypair.json +$solana_keygen new -f -o "$SOLANA_CONFIG_DIR"/mint-keypair.json -mkdir "$SOLANA_CONFIG_DIR"/bootstrap-leader $solana_keygen new -o "$SOLANA_CONFIG_DIR"/bootstrap-leader/identity-keypair.json $solana_keygen new -o "$SOLANA_CONFIG_DIR"/bootstrap-leader/vote-keypair.json $solana_keygen new -o "$SOLANA_CONFIG_DIR"/bootstrap-leader/stake-keypair.json diff --git a/net/net.sh b/net/net.sh index 193fbc5b83..16e165dc24 100755 --- a/net/net.sh +++ b/net/net.sh @@ -22,12 +22,11 @@ Operate a configured testnet sanity - Sanity check the network stop - Stop the network restart - Shortcut for stop then start - update - Live update all network nodes logs - Fetch remote logs from each network node startnode- Start an individual node (previously stopped with stopNode) stopnode - Stop an individual node - start/update-specific options: + start-specific options: -T [tarFilename] - Deploy the specified release tarball -t edge|beta|stable|vX.Y.Z - Deploy the latest tarball release for the specified release channel (edge|beta|stable) or release tag @@ -37,7 +36,7 @@ Operate a configured testnet (-t option must be supplied as well) -f [cargoFeatures] - List of |cargo --feaures=| to activate (ignored if -s or -S is specified) - -r - Reuse existing node/ledger configuration from a + -r / --skip-setup - Reuse existing node/ledger configuration from a previous |start| (ie, don't run ./multinode-demo/setup.sh). -d / --debug - Build/deploy the testnet with debug binaries -D /path/to/programs - Deploy custom programs from this location @@ -66,7 +65,8 @@ Operate a configured testnet --internal-nodes-lamports NUM_LAMPORTS_PER_NODE - Amount to fund internal nodes in genesis block. --external-accounts-file FILE_PATH - - A YML file with a list of account pubkeys and corresponding lamport balances in genesis block for external nodes + - A YML file with a list of account pubkeys and corresponding lamport balances + in genesis block for external nodes --no-snapshot-fetch - If set, disables booting validators from a snapshot --skip-ledger-verify @@ -81,7 +81,7 @@ Operate a configured testnet existing binaries - sanity/start/update-specific options: + sanity/start-specific options: -F - Discard validator nodes that didn't bootup successfully -o noValidatorSanity - Skip fullnode sanity -o noInstallCheck - Skip solana-install sanity @@ -107,7 +107,6 @@ deployMethod=local sanityExtraArgs= cargoFeatures= skipSetup=false -updateNodes=false customPrograms= updatePlatforms= nodeAddress= @@ -160,6 +159,9 @@ while [[ -n $1 ]]; do elif [[ $1 = --skip-ledger-verify ]]; then maybeSkipLedgerVerify="$1" shift 1 + elif [[ $1 = --skip-setup ]]; then + skipSetup=true + shift 1 elif [[ $1 = --deploy-update ]]; then updatePlatforms="$updatePlatforms $2" shift 2 @@ -356,12 +358,12 @@ startCommon() { if $skipSetup; then ssh "${sshOptions[@]}" "$ipAddress" " set -x; - mkdir -p ~/solana/config{,-local} - rm -rf ~/config{,-local}; - mv ~/solana/config{,-local} ~; + mkdir -p ~/solana/config; + rm -rf ~/config; + mv ~/solana/config ~; rm -rf ~/solana; mkdir -p ~/solana ~/.cargo/bin; - mv ~/config{,-local} ~/solana/ + mv ~/config ~/solana/ " else ssh "${sshOptions[@]}" "$ipAddress" " @@ -646,11 +648,7 @@ prepare_deploy() { deploy() { echo "Deployment started at $(date)" - if $updateNodes; then - $metricsWriteDatapoint "testnet-deploy net-update-begin=1" - else - $metricsWriteDatapoint "testnet-deploy net-start-begin=1" - fi + $metricsWriteDatapoint "testnet-deploy net-start-begin=1" declare bootstrapLeader=true for nodeAddress in "${fullnodeIpList[@]}" "${blockstreamerIpList[@]}" "${replicatorIpList[@]}"; do @@ -699,11 +697,6 @@ deploy() { annotateBlockexplorerUrl - if $updateNodes; then - for ipAddress in "${clientIpList[@]}"; do - stopNode "$ipAddress" true - done - fi sanity skipBlockstreamerSanity # skip sanity on blockstreamer node, it may not # have caught up to the bootstrap leader yet @@ -717,11 +710,7 @@ deploy() { done clientDeployTime=$SECONDS - if $updateNodes; then - $metricsWriteDatapoint "testnet-deploy net-update-complete=1" - else - $metricsWriteDatapoint "testnet-deploy net-start-complete=1" - fi + $metricsWriteDatapoint "testnet-deploy net-start-complete=1" declare networkVersion=unknown case $deployMethod in @@ -768,7 +757,7 @@ stopNode() { PS4=\"$PS4\" set -x ! tmux list-sessions || tmux kill-session - for pid in solana/{net-stats,fd-monitor,oom-monitor}.pid; do + for pid in solana/{blockexplorer,net-stats,fd-monitor,oom-monitor}.pid; do pgid=\$(ps opgid= \$(cat \$pid) | tr -d '[:space:]') if [[ -n \$pgid ]]; then sudo kill -- -\$pgid @@ -824,11 +813,6 @@ start) prepare_deploy deploy ;; -update) - skipSetup=true - updateNodes=true - start - ;; sanity) sanity ;; diff --git a/net/remote/remote-client.sh b/net/remote/remote-client.sh index a281985a6f..397d757235 100755 --- a/net/remote/remote-client.sh +++ b/net/remote/remote-client.sh @@ -50,7 +50,7 @@ esac case $clientToRun in solana-bench-tps) net/scripts/rsync-retry.sh -vPrc \ - "$entrypointIp":~/solana/solana-client-accounts/bench-tps"$clientIndex".yml ./client-accounts.yml + "$entrypointIp":~/solana/config/bench-tps"$clientIndex".yml ./client-accounts.yml clientCommand="\ solana-bench-tps \ --entrypoint $entrypointIp:8001 \ @@ -65,7 +65,7 @@ solana-bench-tps) solana-bench-exchange) solana-keygen new -f -o bench.keypair net/scripts/rsync-retry.sh -vPrc \ - "$entrypointIp":~/solana/solana-client-accounts/bench-exchange"$clientIndex".yml ./client-accounts.yml + "$entrypointIp":~/solana/config/bench-exchange"$clientIndex".yml ./client-accounts.yml clientCommand="\ solana-bench-exchange \ --entrypoint $entrypointIp:8001 \ @@ -97,11 +97,11 @@ echo "$(date) | $0 $*" >> client.log ( sudo SOLANA_METRICS_CONFIG="$SOLANA_METRICS_CONFIG" scripts/oom-monitor.sh ) > oom-monitor.log 2>&1 & -echo $! > oom-monitor.pid +echo \$! > oom-monitor.pid scripts/fd-monitor.sh > fd-monitor.log 2>&1 & -echo $! > fd-monitor.pid +echo \$! > fd-monitor.pid scripts/net-stats.sh > net-stats.log 2>&1 & -echo $! > net-stats.pid +echo \$! > net-stats.pid ! tmux list-sessions || tmux kill-session tmux new -s "$clientToRun" -d " diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index ce7435e172..d50daee32f 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -107,11 +107,11 @@ cat >> ~/solana/on-reboot < oom-monitor.log 2>&1 & - echo $! > oom-monitor.pid + echo \$! > oom-monitor.pid scripts/fd-monitor.sh > fd-monitor.log 2>&1 & - echo $! > fd-monitor.pid + echo \$! > fd-monitor.pid scripts/net-stats.sh > net-stats.log 2>&1 & - echo $! > net-stats.pid + echo \$! > net-stats.pid if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-validator-cuda ]]; then echo Selecting solana-validator-cuda @@ -123,15 +123,14 @@ EOF bootstrap-leader) set -x if [[ $skipSetup != true ]]; then - rm -rf ./solana-node-keys - rm -rf ./solana-node-balances - mkdir ./solana-node-balances + multinode-demo/clear-config.sh + if [[ -n $internalNodesLamports ]]; then - echo "---" >> ./solana-node-balances/fullnode-balances.yml + echo "---" >> config/fullnode-balances.yml for i in $(seq 0 "$numNodes"); do - solana-keygen new -o ./solana-node-keys/"$i" - pubkey="$(solana-keygen pubkey ./solana-node-keys/"$i")" - cat >> ./solana-node-balances/fullnode-balances.yml <> config/fullnode-balances.yml <> ./solana-client-accounts/client-accounts.yml - echo "" >> ./solana-client-accounts/client-accounts.yml + tail -n +2 -q config/bench-tps"$i".yml >> config/client-accounts.yml + echo "" >> config/client-accounts.yml done for i in $(seq 0 $((numBenchExchangeClients-1))); do # shellcheck disable=SC2086 # Do not want to quote $benchExchangeExtraArgs solana-bench-exchange --batch-size 1000 --fund-amount 20000 \ - --write-client-keys ./solana-client-accounts/bench-exchange"$i".yml $benchExchangeExtraArgs - tail -n +2 -q ./solana-client-accounts/bench-exchange"$i".yml >> ./solana-client-accounts/client-accounts.yml - echo "" >> ./solana-client-accounts/client-accounts.yml + --write-client-keys config/bench-exchange"$i".yml $benchExchangeExtraArgs + tail -n +2 -q config/bench-exchange"$i".yml >> config/client-accounts.yml + echo "" >> config/client-accounts.yml done - [[ -z $externalPrimordialAccountsFile ]] || cat "$externalPrimordialAccountsFile" >> ./solana-node-balances/fullnode-balances.yml - if [ -f ./solana-node-balances/fullnode-balances.yml ]; then - genesisOptions+=" --primordial-accounts-file ./solana-node-balances/fullnode-balances.yml" + if [[ -f $externalPrimordialAccountsFile ]]; then + cat "$externalPrimordialAccountsFile" >> config/fullnode-balances.yml fi - if [ -f ./solana-client-accounts/client-accounts.yml ]; then - genesisOptions+=" --primordial-keypairs-file ./solana-client-accounts/client-accounts.yml" + if [[ -f config/fullnode-balances.yml ]]; then + genesisOptions+=" --primordial-accounts-file config/fullnode-balances.yml" + fi + if [[ -f config/client-accounts.yml ]]; then + genesisOptions+=" --primordial-keypairs-file config/client-accounts.yml" fi args=( @@ -184,7 +183,7 @@ EOF fi # shellcheck disable=SC2206 # Do not want to quote $genesisOptions args+=($genesisOptions) - ./multinode-demo/setup.sh "${args[@]}" + multinode-demo/setup.sh "${args[@]}" fi args=( --gossip-port "$entrypointIp":8001 @@ -208,18 +207,20 @@ EOF ~/solana/on-reboot waitForNodeToInit - solana --url http://"$entrypointIp":8899 \ - --keypair ~/solana/config/bootstrap-leader/identity-keypair.json \ - validator-info publish "$(hostname)" -n team/solana --force || true + if [[ $skipSetup != true ]]; then + solana --url http://"$entrypointIp":8899 \ + --keypair ~/solana/config/bootstrap-leader/identity-keypair.json \ + validator-info publish "$(hostname)" -n team/solana --force || true + fi ;; validator|blockstreamer) if [[ $deployMethod != skip ]]; then net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/ fi if [[ $skipSetup != true ]]; then - rm -f ~/solana/fullnode-identity.json + multinode-demo/clear-config.sh [[ -z $internalNodesLamports ]] || net/scripts/rsync-retry.sh -vPrc \ - "$entrypointIp":~/solana/solana-node-keys/"$nodeIndex" ~/solana/fullnode-identity.json + "$entrypointIp":~/solana/config/fullnode-"$nodeIndex"-identity.json config/fullnode-identity.json fi args=( @@ -239,20 +240,16 @@ EOF fi fi - if [[ ! -f ~/solana/fullnode-identity.json ]]; then - solana-keygen new -o ~/solana/fullnode-identity.json + if [[ ! -f config/fullnode-identity.json ]]; then + solana-keygen new -o config/fullnode-identity.json fi - args+=(--identity ~/solana/fullnode-identity.json) + args+=(--identity config/fullnode-identity.json) if [[ $airdropsEnabled != true ]]; then args+=(--no-airdrop) fi set -x - if [[ $skipSetup != true ]]; then - ./multinode-demo/clear-config.sh - fi - if [[ $nodeType = blockstreamer ]]; then # Sneak the mint-keypair.json from the bootstrap leader and run another drone # with it on the blockstreamer node. Typically the blockstreamer node has @@ -261,7 +258,7 @@ EOF scp "$entrypointIp":~/solana/config/mint-keypair.json config/ if [[ $airdropsEnabled = true ]]; then cat >> ~/solana/on-reboot < drone.log 2>&1 & + multinode-demo/drone.sh > drone.log 2>&1 & EOF fi @@ -278,6 +275,7 @@ EOF killall node || true export BLOCKEXPLORER_GEOIP_WHITELIST=$PWD/net/config/geoip.yml npx solana-blockexplorer > blockexplorer.log 2>&1 & + echo \$! > blockexplorer.pid # Redirect port 80 to port 5000 sudo iptables -A INPUT -p tcp --dport 80 -j ACCEPT @@ -301,7 +299,7 @@ EOF # shellcheck disable=SC2206 # Don't want to double quote $extraNodeArgs args+=($extraNodeArgs) cat >> ~/solana/on-reboot < fullnode.log 2>&1 & + nohup multinode-demo/validator.sh ${args[@]} > fullnode.log 2>&1 & pid=\$! oom_score_adj "\$pid" 1000 disown @@ -318,16 +316,18 @@ EOF if [[ $airdropsEnabled != true ]]; then args+=(--no-airdrop) fi - if [[ -f ~/solana/fullnode-identity.json ]]; then - args+=(--keypair ~/solana/fullnode-identity.json) + if [[ -f config/fullnode-identity.json ]]; then + args+=(--keypair config/fullnode-identity.json) fi - ./multinode-demo/delegate-stake.sh "${args[@]}" + multinode-demo/delegate-stake.sh "${args[@]}" fi - solana --url http://"$entrypointIp":8899 \ - --keypair ~/solana/fullnode-identity.json \ - validator-info publish "$(hostname)" -n team/solana --force || true + if [[ $skipSetup != true ]]; then + solana --url http://"$entrypointIp":8899 \ + --keypair config/fullnode-identity.json \ + validator-info publish "$(hostname)" -n team/solana --force || true + fi ;; replicator) if [[ $deployMethod != skip ]]; then @@ -346,7 +346,7 @@ EOF fi cat >> ~/solana/on-reboot < fullnode.log 2>&1 & + nohup multinode-demo/replicator.sh ${args[@]} > fullnode.log 2>&1 & pid=\$! oom_score_adj "\$pid" 1000 disown