Refactor multinode-demo/ scripts to avoid shipping fullnode-x.sh (#3835)

book/src/testnet-participation.md

@@ -91,7 +91,6 @@ $ export PATH=$PWD/bin:$PATH
 ```
 
 ### Starting The Validator
-
 Sanity check that you are able to interact with the cluster by receiving a small
 airdrop of lamports from the testnet drone:
 ```bash
@@ -103,34 +102,52 @@ Then the following command will start a new validator node.
 
 If this is a `solana-install`-installation:
 ```bash
-$ fullnode-x.sh --public-address --poll-for-new-genesis-block beta.testnet.solana.com
+$ clear-fullnode-config.sh
+$ fullnode.sh --public-address --poll-for-new-genesis-block beta.testnet.solana.com
 ```
 
 Alternatively, the `solana-install run` command can be used to run the validator
 node while periodically checking for and applying software updates:
 ```bash
-$ solana-install run fullnode-x.sh -- --public-address --poll-for-new-genesis-block beta.testnet.solana.com
+$ solana-install run fullnode.sh -- --public-address --poll-for-new-genesis-block beta.testnet.solana.com
 ```
 
-When not using `solana-install`:
+If you built from source:
 ```bash
-$ USE_INSTALL=1 ./multinode-demo/fullnode-x.sh --public-address --poll-for-new-genesis-block beta.testnet.solana.com
+$ USE_INSTALL=1 ./multinode-demo/clear-fullnode-config.sh
+$ USE_INSTALL=1 ./multinode-demo/fullnode.sh --public-address --poll-for-new-genesis-block edge.testnet.solana.com
 ```
 
-Then from another console, confirm the IP address if your node is now visible in
-the gossip network by running:
-```bash
-$ solana-gossip --network beta.testnet.solana.com:8001
-```
-
-Congratulations, you're now participating in the testnet cluster!
-
 #### Controlling local network port allocation
 By default the validator will dynamically select available network ports in the
 8000-10000 range, and may be overridden with `--dynamic-port-range`.  For
-example, `fullnode-x.sh --dynamic-port-range 11000-11010 ...` will restrict the
+example, `fullnode.sh --dynamic-port-range 11000-11010 ...` will restrict the
 validator to ports 11000-11011.
 
+### Validator Monitoring
+From another console, confirm the IP address of your validator is visible in the
+gossip network by running:
+```bash
+solana-gossip --network edge.testnet.solana.com:8001
+```
+
+When `fullnode.sh` starts, it will output a fullnode configuration that looks
+similar to:
+```bash
+======================[ Fullnode configuration ]======================
+node id: 4ceWXsL3UJvn7NYZiRkw7NsryMpviaKBDYr8GK7J61Dm
+vote id: 2ozWvfaXQd1X6uKh8jERoRGApDqSqcEy6fF1oN13LL2G
+ledger: ...
+accounts: ...
+======================================================================
+```
+
+Provide the **vote id** pubkey to the `solana-wallet show-vote-account` command to view
+the recent voting activity from your validator:
+```bash
+$ solana-wallet -n beta.testnet.solana.com show-vote-account 2ozWvfaXQd1X6uKh8jERoRGApDqSqcEy6fF1oN13LL2G
+```
+
 ### Sharing Metrics From Your Validator
 If you'd like to share metrics perform the following steps before starting the
 validator node:

ci/localnet-sanity.sh

@@ -90,7 +90,7 @@ nodes=(
 for i in $(seq 1 $extraNodes); do
   nodes+=(
     "multinode-demo/fullnode.sh \
-      -X dyn$i \
+      --label dyn$i \
       --init-complete-file init-complete-node$((2 + i)).log \
       $maybeNoLeaderRotation"
   )

ci/publish-tarball.sh

@@ -77,16 +77,16 @@ exec multinode-demo/fullnode.sh "$@"
 EOF
   chmod +x solana-release/bin/fullnode.sh
 
-  # Add a wrapper script for fullnode-x.sh
+  # Add a wrapper script for clear-fullnode-config.sh
   # TODO: Remove multinode/... from tarball
-  cat > solana-release/bin/fullnode-x.sh <<'EOF'
+  cat > solana-release/bin/clear-fullnode-config.sh <<'EOF'
 #!/usr/bin/env bash
 set -e
 cd "$(dirname "$0")"/..
 export USE_INSTALL=1
-exec multinode-demo/fullnode-x.sh "$@"
+exec multinode-demo/clear-fullnode-config.sh "$@"
 EOF
-  chmod +x solana-release/bin/fullnode-x.sh
+  chmod +x solana-release/bin/clear-fullnode-config.sh
 
   tar jvcf solana-release-$TARGET.tar.bz2 solana-release/
   cp solana-release/bin/solana-install solana-install-$TARGET

multinode-demo/clear-fullnode-config.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+#
+# Clear the current cluster configuration
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+set -e
+
+for i in "$SOLANA_RSYNC_CONFIG_DIR" "$SOLANA_CONFIG_DIR"; do
+  echo "Cleaning $i"
+  rm -rvf "$i"
+  mkdir -p "$i"
+done
+

multinode-demo/common.sh

@@ -7,7 +7,7 @@
 # shellcheck disable=2034
 #
 
-solana_root="$(dirname "${BASH_SOURCE[0]}")/.."
+SOLANA_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. || exit 1; pwd)"
 
 rsync=rsync
 bootstrap_leader_logger="tee bootstrap-leader.log"
@@ -23,8 +23,7 @@ if [[ $(uname) != Linux ]]; then
   fi
 fi
 
-
+if [[ -n $USE_INSTALL || ! -f "$SOLANA_ROOT"/Cargo.toml ]]; then
-if [[ -n $USE_INSTALL || ! -f "$solana_root"/Cargo.toml ]]; then
   solana_program() {
     declare program="$1"
     printf "solana-%s" "$program"
@@ -38,17 +37,17 @@ else
       features+="cuda,"
     fi
 
-    if [[ -r "$solana_root"/"$program"/Cargo.toml ]]; then
+    if [[ -r "$SOLANA_ROOT/$program"/Cargo.toml ]]; then
       maybe_package="--package solana-$program"
     fi
     if [[ -n $NDEBUG ]]; then
       maybe_release=--release
     fi
-    declare manifest_path="--manifest-path=$solana_root/$program/Cargo.toml"
+    declare manifest_path="--manifest-path=$SOLANA_ROOT/$program/Cargo.toml"
     printf "cargo run $manifest_path $maybe_release $maybe_package --bin solana-%s %s -- " "$program" "$features"
   }
   # shellcheck disable=2154 # 'here' is referenced but not assigned
-  LD_LIBRARY_PATH=$(cd "$here/../target/perf-libs" && pwd):$LD_LIBRARY_PATH
+  LD_LIBRARY_PATH=$(cd "$SOLANA_ROOT/target/perf-libs" && pwd):$LD_LIBRARY_PATH
   export LD_LIBRARY_PATH
 fi
 
@@ -66,14 +65,14 @@ export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
 export RUST_BACKTRACE=1
 
 # shellcheck source=scripts/configure-metrics.sh
-source "$solana_root"/scripts/configure-metrics.sh
+source "$SOLANA_ROOT"/scripts/configure-metrics.sh
 
 tune_system() {
   # Skip in CI
   [[ -z $CI ]] || return 0
 
   # shellcheck source=scripts/ulimit-n.sh
-  source "$solana_root"/scripts/ulimit-n.sh
+  source "$SOLANA_ROOT"/scripts/ulimit-n.sh
 
   # Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360
   if [[ $(uname) = Linux ]]; then
@@ -107,76 +106,20 @@ tune_system() {
   fi
 }
 
-airdrop() {
-  declare keypair_file=$1
-  declare host=$2
-  declare amount=$3
-
-  declare address
-  address=$($solana_wallet --keypair "$keypair_file" address)
-
-  # TODO: Until https://github.com/solana-labs/solana/issues/2355 is resolved
-  # a fullnode needs N lamports as its vote account gets re-created on every
-  # node restart, costing it lamports
-  declare retries=5
-
-  while ! $solana_wallet --keypair "$keypair_file" --host "$host" airdrop "$amount"; do
-
-    # TODO: Consider moving this retry logic into `solana-wallet airdrop`
-    #   itself, currently it does not retry on "Connection refused" errors.
-    ((retries--))
-    if [[ $retries -le 0 ]]; then
-        echo "Airdrop to $address failed."
-        return 1
-    fi
-    echo "Airdrop to $address failed. Remaining retries: $retries"
-    sleep 1
-  done
-
-  return 0
-}
-
-setup_vote_account() {
-  declare drone_address=$1
-  declare node_id_path=$2
-  declare vote_id_path=$3
-  declare stake=$4
-
-  declare node_id
-  node_id=$($solana_wallet --keypair "$node_id_path" address)
-
-  declare vote_id
-  vote_id=$($solana_wallet --keypair "$vote_id_path" address)
-
-  if [[ -f "$vote_id_path".configured ]]; then
-    echo "Vote account has already been configured"
-  else
-    airdrop "$node_id_path" "$drone_address" "$stake" || return $?
-
-    # Fund the vote account from the node, with the node as the node_id
-    $solana_wallet --keypair "$node_id_path" --host "$drone_address" \
-      create-vote-account "$vote_id" "$node_id" $((stake - 1)) || return $?
-  fi
-
-  $solana_wallet --keypair "$node_id_path" --host "$drone_address" show-vote-account "$vote_id"
-  return 0
-}
-
 fullnode_usage() {
   if [[ -n $1 ]]; then
     echo "$*"
     echo
   fi
   cat <<EOF
-usage: $0 [-x] [--blockstream PATH] [--init-complete-file FILE] [--stake LAMPORTS] [--no-voting] [--rpc-port port] [rsync network path to bootstrap leader configuration] [network entry point]
+usage: $0 [--blockstream PATH] [--init-complete-file FILE] [--label LABEL] [--stake LAMPORTS] [--no-voting] [--rpc-port port] [rsync network path to bootstrap leader configuration] [network entry point]
 
-Start a full node on the specified network
+Start a full node
 
-  -x                        - start a new, dynamically-configured full node. Does not apply to the bootstrap leader
-  -X [label]                - start or restart a dynamically-configured full node with
-                              the specified label. Does not apply to the bootstrap leader
   --blockstream PATH        - open blockstream at this unix domain socket location
   --init-complete-file FILE - create this file, if it doesn't already exist, once node initialization is complete
+  --label LABEL             - Append the given label to the fullnode configuration files, useful when running
+                              multiple fullnodes from the same filesystem location
   --stake LAMPORTS          - Number of lamports to stake
   --public-address          - advertise public machine address in gossip.  By default the local machine address is advertised
   --no-voting               - start node without vote signer
@@ -186,9 +129,8 @@ EOF
   exit 1
 }
 
-# The directory on the bootstrap leader that is rsynced by other full nodes as
+# The directory on the cluster entrypoint that is rsynced by other full nodes
-# they boot (TODO: Eventually this should go away)
+SOLANA_RSYNC_CONFIG_DIR=$SOLANA_ROOT/config
-SOLANA_RSYNC_CONFIG_DIR=$PWD/config
 
 # Configuration that remains local
-SOLANA_CONFIG_DIR=$PWD/config-local
+SOLANA_CONFIG_DIR=$SOLANA_ROOT/config-local

multinode-demo/fullnode-x.sh

@@ -5,4 +5,4 @@
 
 here=$(dirname "$0")
 
-exec "$here"/fullnode.sh -x "$@"
+exec "$here"/fullnode.sh --label x$$ "$@"

multinode-demo/fullnode.sh

@@ -13,21 +13,15 @@ if [[ $1 = -h ]]; then
   fullnode_usage "$@"
 fi
 
-gossip_port=
 extra_fullnode_args=()
-self_setup=0
 stake=43 # number of lamports to assign as stake (plus transaction fee to setup the stake)
 poll_for_new_genesis_block=0
+label=
 
 while [[ ${1:0:1} = - ]]; do
-  if [[ $1 = -X ]]; then
+  if [[ $1 = --label ]]; then
-    self_setup=1
+    label="-$2"
-    self_setup_label=$2
     shift 2
-  elif [[ $1 = -x ]]; then
-    self_setup=1
-    self_setup_label=$$
-    shift
   elif [[ $1 = --poll-for-new-genesis-block ]]; then
     poll_for_new_genesis_block=1
     shift
@@ -51,7 +45,6 @@ while [[ ${1:0:1} = - ]]; do
     extra_fullnode_args+=("$1")
     shift
   elif [[ $1 = --gossip-port ]]; then
-    gossip_port=$2
     extra_fullnode_args+=("$1" "$2")
     shift 2
   elif [[ $1 = --rpc-port ]]; then
@@ -99,37 +92,26 @@ else
   program=$solana_fullnode
 fi
 
-if ((!self_setup)); then
+fullnode_id_path=$SOLANA_CONFIG_DIR/fullnode-id$label.json
-  [[ -f $SOLANA_CONFIG_DIR/fullnode-id.json ]] || {
+fullnode_vote_id_path=$SOLANA_CONFIG_DIR/fullnode-vote-id$label.json
-    echo "$SOLANA_CONFIG_DIR/fullnode-id.json not found, create it by running:"
+ledger_config_dir=$SOLANA_CONFIG_DIR/fullnode-ledger$label
-    echo
+accounts_config_dir=$SOLANA_CONFIG_DIR/fullnode-accounts$label
-    echo "  ${here}/setup.sh"
-    exit 1
-  }
-  fullnode_id_path=$SOLANA_CONFIG_DIR/fullnode-id.json
-  fullnode_vote_id_path=$SOLANA_CONFIG_DIR/fullnode-vote-id.json
-  ledger_config_dir=$SOLANA_CONFIG_DIR/fullnode-ledger
-  accounts_config_dir=$SOLANA_CONFIG_DIR/fullnode-accounts
 
-  if [[ -z $gossip_port ]]; then
+mkdir -p "$SOLANA_CONFIG_DIR"
-    extra_fullnode_args+=("--gossip-port" 9000)
+[[ -r "$fullnode_id_path" ]] || $solana_keygen -o "$fullnode_id_path"
-  fi
+[[ -r "$fullnode_vote_id_path" ]] || $solana_keygen -o "$fullnode_vote_id_path"
-else
-  mkdir -p "$SOLANA_CONFIG_DIR"
-  fullnode_id_path=$SOLANA_CONFIG_DIR/fullnode-id-x$self_setup_label.json
-  fullnode_vote_id_path=$SOLANA_CONFIG_DIR/fullnode-vote-id-x$self_setup_label.json
-  [[ -f "$fullnode_id_path" ]] || $solana_keygen -o "$fullnode_id_path"
-  [[ -f "$fullnode_vote_id_path" ]] || $solana_keygen -o "$fullnode_vote_id_path"
-  ledger_config_dir=$SOLANA_CONFIG_DIR/fullnode-ledger-x$self_setup_label
-  accounts_config_dir=$SOLANA_CONFIG_DIR/fullnode-accounts-x$self_setup_label
-fi
 
+fullnode_id=$($solana_keygen pubkey "$fullnode_id_path")
 fullnode_vote_id=$($solana_keygen pubkey "$fullnode_vote_id_path")
 
-[[ -r $fullnode_id_path ]] || {
+cat <<EOF
-  echo "$fullnode_id_path does not exist"
+======================[ Fullnode configuration ]======================
-  exit 1
+node id: $fullnode_id
-}
+vote id: $fullnode_vote_id
+ledger: $ledger_config_dir
+accounts: $accounts_config_dir
+======================================================================
+EOF
 
 tune_system
 
@@ -152,10 +134,65 @@ rsync_url() { # adds the 'rsync://` prefix to URLs that need it
   echo "rsync://$url"
 }
 
+airdrop() {
+  declare keypair_file=$1
+  declare host=$2
+  declare amount=$3
+
+  declare address
+  address=$($solana_wallet --keypair "$keypair_file" address)
+
+  # TODO: Until https://github.com/solana-labs/solana/issues/2355 is resolved
+  # a fullnode needs N lamports as its vote account gets re-created on every
+  # node restart, costing it lamports
+  declare retries=5
+
+  while ! $solana_wallet --keypair "$keypair_file" --host "$host" airdrop "$amount"; do
+
+    # TODO: Consider moving this retry logic into `solana-wallet airdrop`
+    #   itself, currently it does not retry on "Connection refused" errors.
+    ((retries--))
+    if [[ $retries -le 0 ]]; then
+        echo "Airdrop to $address failed."
+        return 1
+    fi
+    echo "Airdrop to $address failed. Remaining retries: $retries"
+    sleep 1
+  done
+
+  return 0
+}
+
+setup_vote_account() {
+  declare drone_address=$1
+  declare node_id_path=$2
+  declare vote_id_path=$3
+  declare stake=$4
+
+  declare node_id
+  node_id=$($solana_wallet --keypair "$node_id_path" address)
+
+  declare vote_id
+  vote_id=$($solana_wallet --keypair "$vote_id_path" address)
+
+  if [[ -f "$vote_id_path".configured ]]; then
+    echo "Vote account has already been configured"
+  else
+    airdrop "$node_id_path" "$drone_address" "$stake" || return $?
+
+    # Fund the vote account from the node, with the node as the node_id
+    $solana_wallet --keypair "$node_id_path" --host "$drone_address" \
+      create-vote-account "$vote_id" "$node_id" $((stake - 1)) || return $?
+
+    touch "$vote_id_path".configured
+  fi
+
+  $solana_wallet --keypair "$node_id_path" --host "$drone_address" show-vote-account "$vote_id"
+  return 0
+}
 
-rsync_leader_url=$(rsync_url "$leader")
 set -e
-
+rsync_leader_url=$(rsync_url "$leader")
 secs_to_next_genesis_poll=0
 PS4="$(basename "$0"): "
 while true; do
@@ -204,9 +241,9 @@ while true; do
         $rsync -r "$rsync_leader_url"/config/ledger "$SOLANA_RSYNC_CONFIG_DIR" || true
         if [[ -n $(diff "$SOLANA_RSYNC_CONFIG_DIR"/ledger/genesis.json "$ledger_config_dir"/genesis.json 2>&1) ]]; then
           echo "############## New genesis detected, restarting fullnode ##############"
-          rm -rf "$ledger_config_dir"
           kill "$pid" || true
           wait "$pid" || true
+          rm -rf "$ledger_config_dir" "$accounts_config_dir" "$vote_id_path".configured
           break
         fi
       fi

multinode-demo/leader.sh

@@ -1 +0,0 @@
-bootstrap-leader.sh

multinode-demo/setup.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
-#
-# Creates a fullnode configuration
-#
 
 here=$(dirname "$0")
 # shellcheck source=multinode-demo/common.sh
 source "$here"/common.sh
 
+lamports=1000000000
+bootstrap_leader_lamports=
+
 usage () {
   exitcode=0
   if [[ -n "$1" ]]; then
@@ -14,25 +14,18 @@ usage () {
     echo "Error: $*"
   fi
   cat <<EOF
-usage: $0 [-n lamports] [-l] [-p] [-t node_type]
+usage: $0 [-n lamports] [-b lamports]
 
-Creates a fullnode configuration
+Create a cluster configuration
 
- -n lamports    - Number of lamports to create
+ -n lamports    - Number of lamports to create [default: $lamports]
- -t node_type   - Create configuration files only for this kind of node.  Valid
+ -b lamports    - Override the number of lamports for the bootstrap leader's stake
-                  options are bootstrap-leader or fullnode.  Creates configuration files
-                  for both by default
 
 EOF
   exit $exitcode
 }
 
-lamports=1000000000
+while getopts "h?n:b:" opt; do
-bootstrap_leader=true
-bootstrap_leader_lamports=
-
-fullnode=true
-while getopts "h?n:b:lpt:" opt; do
   case $opt in
   h|\?)
     usage
@@ -44,22 +37,6 @@ while getopts "h?n:b:lpt:" opt; do
   b)
     bootstrap_leader_lamports="$OPTARG"
     ;;
-  t)
-    node_type="$OPTARG"
-    case $OPTARG in
-    bootstrap-leader|leader) # TODO: Remove legacy 'leader' option
-      bootstrap_leader=true
-      fullnode=false
-      ;;
-    fullnode|validator) # TODO: Remove legacy 'validator' option
-      bootstrap_leader=false
-      fullnode=true
-      ;;
-    *)
-      usage "Error: unknown node type: $node_type"
-      ;;
-    esac
-    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -68,42 +45,24 @@ done
 
 
 set -e
+"$here"/clear-fullnode-config.sh
 
-for i in "$SOLANA_RSYNC_CONFIG_DIR" "$SOLANA_CONFIG_DIR"; do
+# Create genesis ledger
-  echo "Cleaning $i"
+$solana_keygen -o "$SOLANA_CONFIG_DIR"/mint-id.json
-  rm -rvf "$i"
+$solana_keygen -o "$SOLANA_CONFIG_DIR"/bootstrap-leader-id.json
-  mkdir -p "$i"
+$solana_keygen -o "$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-id.json
-done
 
-if $bootstrap_leader; then
+args=(
-  # Create genesis configuration
+  --bootstrap-leader-keypair "$SOLANA_CONFIG_DIR"/bootstrap-leader-id.json
-  (
+  --bootstrap-vote-keypair "$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-id.json
-    set -x
+  --ledger "$SOLANA_RSYNC_CONFIG_DIR"/ledger
-    $solana_keygen -o "$SOLANA_CONFIG_DIR"/mint-id.json
+  --mint "$SOLANA_CONFIG_DIR"/mint-id.json
-    $solana_keygen -o "$SOLANA_CONFIG_DIR"/bootstrap-leader-id.json
+  --lamports "$lamports"
-    $solana_keygen -o "$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-id.json
+)
 
-    args=(
+if [[ -n $bootstrap_leader_lamports ]]; then
-      --bootstrap-leader-keypair "$SOLANA_CONFIG_DIR"/bootstrap-leader-id.json
+  args+=(--bootstrap-leader-lamports "$bootstrap_leader_lamports")
-      --bootstrap-vote-keypair "$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-id.json
-      --ledger "$SOLANA_RSYNC_CONFIG_DIR"/ledger
-      --mint "$SOLANA_CONFIG_DIR"/mint-id.json
-      --lamports "$lamports"
-    )
-
-    if [[ -n $bootstrap_leader_lamports ]]; then
-      args+=(--bootstrap-leader-lamports "$bootstrap_leader_lamports")
-    fi
-
-    $solana_genesis "${args[@]}"
-    cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger
-  )
 fi
 
-if $fullnode; then
+$solana_genesis "${args[@]}"
-  (
+cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger
-    set -x
-    $solana_keygen -o "$SOLANA_CONFIG_DIR"/fullnode-id.json
-    $solana_keygen -o "$SOLANA_CONFIG_DIR"/fullnode-vote-id.json
-  )
-fi

multinode-demo/validator-x.sh

@@ -1 +0,0 @@
-fullnode-x.sh

multinode-demo/validator.sh

@@ -1 +0,0 @@
-fullnode.sh

net/remote/remote-node.sh

@@ -77,7 +77,7 @@ local|tar)
     fi
     set -x
     if [[ $skipSetup != true ]]; then
-      ./multinode-demo/setup.sh -t bootstrap-leader -b $stake
+      ./multinode-demo/setup.sh -b $stake
     fi
     ./multinode-demo/drone.sh > drone.log 2>&1 &
 
@@ -122,7 +122,7 @@ local|tar)
 
     set -x
     if [[ $skipSetup != true ]]; then
-      ./multinode-demo/setup.sh -t fullnode
+      ./multinode-demo/clear-fullnode-config.sh
     fi
 
     if [[ $nodeType = blockstreamer ]]; then