gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rework fullnode.sh to recover better from genesis block resets (#4884) (#4888)

Browse Source 
automerge
This commit is contained in:
mergify[bot]
2019-07-01 12:31:53 -07:00
committed by Grimes
parent 462d0cfc6c
commit 6089db2a07
2 changed files with 135 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
multinode-demo/common.sh
View File

@ -88,3 +88,18 @@ default_arg() {
args+=("$name") args+=("$name")
fi fi
} }
replace_arg() {
declare name=$1
declare value=$2
default_arg "$name" "$value"
declare index=0
for arg in "${args[@]}"; do
index=$((index + 1))
if [[ $arg = "$name" ]]; then
args[$index]="$value"
fi
done
}

248
multinode-demo/fullnode.sh
View File

@ -9,6 +9,7 @@ source "$here"/common.sh
# shellcheck source=scripts/oom-score-adj.sh # shellcheck source=scripts/oom-score-adj.sh
source "$here"/../scripts/oom-score-adj.sh source "$here"/../scripts/oom-score-adj.sh
fullnode_usage() { fullnode_usage() {
if [[ -n $1 ]]; then if [[ -n $1 ]]; then
echo "$*" echo "$*"
@ -76,24 +77,8 @@ rsync_url() { # adds the 'rsync://` prefix to URLs that need it
setup_validator_accounts() { setup_validator_accounts() {
declare entrypoint_ip=$1 declare entrypoint_ip=$1
declare node_keypair_path=$2 declare node_lamports=$2
declare vote_keypair_path=$3 declare stake_lamports=$3
declare stake_keypair_path=$4
declare storage_keypair_path=$5
declare node_lamports=$6
declare stake_lamports=$7
declare node_pubkey
node_pubkey=$($solana_keygen pubkey "$node_keypair_path")
declare vote_pubkey
vote_pubkey=$($solana_keygen pubkey "$vote_keypair_path")
declare stake_pubkey
stake_pubkey=$($solana_keygen pubkey "$stake_keypair_path")
declare storage_pubkey
storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path")
if [[ -f $configured_flag ]]; then if [[ -f $configured_flag ]]; then
echo "Vote and stake accounts have already been configured" echo "Vote and stake accounts have already been configured"
@ -101,75 +86,66 @@ setup_validator_accounts() {
if ((airdrops_enabled)); then if ((airdrops_enabled)); then
# Fund the node with enough tokens to fund its Vote, Staking, and Storage accounts # Fund the node with enough tokens to fund its Vote, Staking, and Storage accounts
declare fees=100 # TODO: No hardcoded transaction fees, fetch the current cluster fees declare fees=100 # TODO: No hardcoded transaction fees, fetch the current cluster fees
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop $((node_lamports+stake_lamports+fees)) || return $? $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" airdrop $((node_lamports+stake_lamports+fees)) || return $?
else else
echo "current account balance is " echo "current account balance is "
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $? $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $?
fi fi
# Fund the vote account from the node, with the node as the node_pubkey # Fund the vote account from the node, with the node as the identity_pubkey
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
create-vote-account "$vote_pubkey" "$node_pubkey" 1 --commission 65535 || return $? create-vote-account "$vote_pubkey" "$identity_pubkey" 1 --commission 65535 || return $?
# Fund the stake account from the node, with the node as the node_pubkey # Fund the stake account from the node, with the node as the identity_pubkey
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
create-stake-account "$stake_pubkey" "$stake_lamports" || return $? create-stake-account "$stake_pubkey" "$stake_lamports" || return $?
# Delegate the stake. The transaction fee is paid by the node but the # Delegate the stake. The transaction fee is paid by the node but the
# transaction must be signed by the stake_keypair # transaction must be signed by the stake_keypair
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
delegate-stake "$stake_keypair_path" "$vote_pubkey" "$stake_lamports" || return $? delegate-stake "$stake_keypair_path" "$vote_pubkey" "$stake_lamports" || return $?
# Setup validator storage account # Setup validator storage account
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
create-validator-storage-account "$node_pubkey" "$storage_pubkey" || return $? create-validator-storage-account "$identity_pubkey" "$storage_pubkey" || return $?
touch "$configured_flag" touch "$configured_flag"
fi fi
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
show-vote-account "$vote_pubkey" show-vote-account "$vote_pubkey"
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
show-stake-account "$stake_pubkey" show-stake-account "$stake_pubkey"
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
show-storage-account "$storage_pubkey" show-storage-account "$storage_pubkey"
echo "Identity account balance:" echo "Identity account balance:"
$solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" balance $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" balance
echo "========================================================================"
return 0 return 0
} }
setup_replicator_account() { setup_replicator_account() {
declare entrypoint_ip=$1 declare entrypoint_ip=$1
declare node_keypair_path=$2 declare node_lamports=$2
declare storage_keypair_path=$3
declare node_lamports=$4
declare node_pubkey
node_pubkey=$($solana_keygen pubkey "$node_keypair_path")
declare storage_pubkey
storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path")
if [[ -f $configured_flag ]]; then if [[ -f $configured_flag ]]; then
echo "Replicator account has already been configured" echo "Replicator account has already been configured"
else else
if ((airdrops_enabled)); then if ((airdrops_enabled)); then
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop "$node_lamports" || return $? $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" airdrop "$node_lamports" || return $?
else else
echo "current account balance is " echo "current account balance is "
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $? $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $?
fi fi
# Setup replicator storage account # Setup replicator storage account
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
create-replicator-storage-account "$node_pubkey" "$storage_pubkey" || return $? create-replicator-storage-account "$identity_pubkey" "$storage_pubkey" || return $?
touch "$configured_flag" touch "$configured_flag"
fi fi
$solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \ $solana_wallet --keypair "$identity_keypair_path" --url "http://$entrypoint_ip:8899" \
show-storage-account "$storage_pubkey" show-storage-account "$storage_pubkey"
return 0 return 0
@ -275,24 +251,13 @@ if [[ $node_type = replicator ]]; then
shift "$shift" shift "$shift"
: "${identity_keypair_path:=$SOLANA_CONFIG_DIR/replicator-keypair$label.json}" : "${identity_keypair_path:=$SOLANA_CONFIG_DIR/replicator-keypair$label.json}"
mkdir -p "$SOLANA_CONFIG_DIR"
[[ -r "$identity_keypair_path" ]] || $solana_keygen new -o "$identity_keypair_path"
storage_keypair_path="$SOLANA_CONFIG_DIR"/replicator-storage-keypair$label.json storage_keypair_path="$SOLANA_CONFIG_DIR"/replicator-storage-keypair$label.json
ledger_config_dir=$SOLANA_CONFIG_DIR/replicator-ledger$label ledger_config_dir=$SOLANA_CONFIG_DIR/replicator-ledger$label
configured_flag=$SOLANA_CONFIG_DIR/replicator$label.configured configured_flag=$SOLANA_CONFIG_DIR/replicator$label.configured
mkdir -p "$SOLANA_CONFIG_DIR"
[[ -r "$identity_keypair_path" ]] || $solana_keygen new -o "$identity_keypair_path"
[[ -r "$storage_keypair_path" ]] || $solana_keygen new -o "$storage_keypair_path"
identity_pubkey=$($solana_keygen pubkey "$identity_keypair_path")
storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path")
cat <<EOF
======================[ $node_type configuration ]======================
replicator pubkey: $identity_pubkey
storage pubkey: $storage_pubkey
ledger: $ledger_config_dir
======================================================================
EOF
program=$solana_replicator program=$solana_replicator
default_arg --entrypoint "$entrypoint_address" default_arg --entrypoint "$entrypoint_address"
default_arg --identity "$identity_keypair_path" default_arg --identity "$identity_keypair_path"
@ -300,6 +265,7 @@ EOF
default_arg --ledger "$ledger_config_dir" default_arg --ledger "$ledger_config_dir"
rsync_entrypoint_url=$(rsync_url "$entrypoint") rsync_entrypoint_url=$(rsync_url "$entrypoint")
elif [[ $node_type = bootstrap_leader ]]; then elif [[ $node_type = bootstrap_leader ]]; then
if [[ ${#positional_args[@]} -ne 0 ]]; then if [[ ${#positional_args[@]} -ne 0 ]]; then
fullnode_usage "Unknown argument: ${positional_args[0]}" fullnode_usage "Unknown argument: ${positional_args[0]}"
@ -311,9 +277,11 @@ elif [[ $node_type = bootstrap_leader ]]; then
$solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger verify $solana_ledger_tool --ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger verify
: "${identity_keypair_path:=$SOLANA_CONFIG_DIR/bootstrap-leader-keypair.json}" : "${identity_keypair_path:=$SOLANA_CONFIG_DIR/bootstrap-leader-keypair.json}"
vote_keypair_path="$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-keypair.json vote_keypair_path="$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-keypair.json
ledger_config_dir="$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger ledger_config_dir="$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger
state_dir="$SOLANA_CONFIG_DIR"/bootstrap-leader-state state_dir="$SOLANA_CONFIG_DIR"/bootstrap-leader-state
stake_keypair_path=$SOLANA_CONFIG_DIR/bootstrap-leader-stake-keypair.json
storage_keypair_path=$SOLANA_CONFIG_DIR/bootstrap-leader-storage-keypair.json storage_keypair_path=$SOLANA_CONFIG_DIR/bootstrap-leader-storage-keypair.json
configured_flag=$SOLANA_CONFIG_DIR/bootstrap-leader.configured configured_flag=$SOLANA_CONFIG_DIR/bootstrap-leader.configured
@ -332,19 +300,16 @@ elif [[ $node_type = validator ]]; then
shift "$shift" shift "$shift"
: "${identity_keypair_path:=$SOLANA_CONFIG_DIR/validator-keypair$label.json}" : "${identity_keypair_path:=$SOLANA_CONFIG_DIR/validator-keypair$label.json}"
mkdir -p "$SOLANA_CONFIG_DIR"
[[ -r "$identity_keypair_path" ]] || $solana_keygen new -o "$identity_keypair_path"
vote_keypair_path=$SOLANA_CONFIG_DIR/validator-vote-keypair$label.json vote_keypair_path=$SOLANA_CONFIG_DIR/validator-vote-keypair$label.json
ledger_config_dir=$SOLANA_CONFIG_DIR/validator-ledger$label ledger_config_dir=$SOLANA_CONFIG_DIR/validator-ledger$label
state_dir="$SOLANA_CONFIG_DIR"/validator-state$label state_dir="$SOLANA_CONFIG_DIR"/validator-state$label
storage_keypair_path=$SOLANA_CONFIG_DIR/validator-storage-keypair$label.json
stake_keypair_path=$SOLANA_CONFIG_DIR/validator-stake-keypair$label.json stake_keypair_path=$SOLANA_CONFIG_DIR/validator-stake-keypair$label.json
storage_keypair_path=$SOLANA_CONFIG_DIR/validator-storage-keypair$label.json
configured_flag=$SOLANA_CONFIG_DIR/validator$label.configured configured_flag=$SOLANA_CONFIG_DIR/validator$label.configured
mkdir -p "$SOLANA_CONFIG_DIR"
[[ -r "$identity_keypair_path" ]] || $solana_keygen new -o "$identity_keypair_path"
[[ -r "$vote_keypair_path" ]] || $solana_keygen new -o "$vote_keypair_path"
[[ -r "$stake_keypair_path" ]] || $solana_keygen new -o "$stake_keypair_path"
[[ -r "$storage_keypair_path" ]] || $solana_keygen new -o "$storage_keypair_path"
default_arg --entrypoint "$entrypoint_address" default_arg --entrypoint "$entrypoint_address"
if ((airdrops_enabled)); then if ((airdrops_enabled)); then
default_arg --rpc-drone-address "${entrypoint_address%:*}:9900" default_arg --rpc-drone-address "${entrypoint_address%:*}:9900"
@ -356,29 +321,14 @@ else
exit 1 exit 1
fi fi
identity_pubkey=$($solana_keygen pubkey "$identity_keypair_path")
if [[ $node_type != replicator ]]; then if [[ $node_type != replicator ]]; then
accounts_config_dir="$state_dir"/accounts accounts_config_dir="$state_dir"/accounts
snapshot_config_dir="$state_dir"/snapshots snapshot_config_dir="$state_dir"/snapshots
identity_pubkey=$($solana_keygen pubkey "$identity_keypair_path")
vote_pubkey=$($solana_keygen pubkey "$vote_keypair_path")
storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path")
cat <<EOF
======================[ $node_type configuration ]======================
identity pubkey: $identity_pubkey
vote pubkey: $vote_pubkey
storage pubkey: $storage_pubkey
ledger: $ledger_config_dir
accounts: $accounts_config_dir
snapshots: $snapshot_config_dir
========================================================================
EOF
default_arg --identity "$identity_keypair_path" default_arg --identity "$identity_keypair_path"
default_arg --voting-keypair "$vote_keypair_path" default_arg --voting-keypair "$vote_keypair_path"
default_arg --vote-account "$vote_pubkey"
default_arg --storage-keypair "$storage_keypair_path" default_arg --storage-keypair "$storage_keypair_path"
default_arg --ledger "$ledger_config_dir" default_arg --ledger "$ledger_config_dir"
default_arg --accounts "$accounts_config_dir" default_arg --accounts "$accounts_config_dir"
@ -397,6 +347,13 @@ if [[ -z $CI ]]; then # Skip in CI
fi fi
new_gensis_block() { new_gensis_block() {
(
set -x
$rsync -r "${rsync_entrypoint_url:?}"/config/ledger "$SOLANA_RSYNC_CONFIG_DIR"
) || (
echo "Error: failed to rsync genesis ledger"
)
! diff -q "$SOLANA_RSYNC_CONFIG_DIR"/ledger/genesis.bin "$ledger_config_dir"/genesis.bin >/dev/null 2>&1 ! diff -q "$SOLANA_RSYNC_CONFIG_DIR"/ledger/genesis.bin "$ledger_config_dir"/genesis.bin >/dev/null 2>&1
} }
@ -417,63 +374,104 @@ kill_fullnode() {
trap 'kill_fullnode' INT TERM ERR trap 'kill_fullnode' INT TERM ERR
while true; do while true; do
if new_gensis_block; then if [[ $node_type != bootstrap_leader ]] && new_gensis_block; then
# If the genesis block has changed remove the now stale ledger and vote # If the genesis block has changed remove the now stale ledger and
# keypair for the node and start all over again # vote/stake/storage keypairs for the node and start all over again
( (
set -x set -x
rm -rf "$ledger_config_dir" "$state_dir" "$configured_flag" rm -rf "$ledger_config_dir" "$state_dir" "$configured_flag"
) )
if [[ $node_type = validator ]]; then
$solana_keygen new -f -o "$vote_keypair_path"
$solana_keygen new -f -o "$stake_keypair_path"
$solana_keygen new -f -o "$storage_keypair_path"
fi
if [[ $node_type = replicator ]]; then
$solana_keygen new -f -o "$storage_keypair_path"
fi
fi fi
if [[ ! -d "$SOLANA_RSYNC_CONFIG_DIR"/ledger ]]; then if [[ $node_type = replicator ]]; then
if [[ $node_type = bootstrap_leader ]]; then storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path")
setup_replicator_account "${entrypoint_address%:*}" \
"$node_lamports"
cat <<EOF
======================[ $node_type configuration ]======================
replicator pubkey: $identity_pubkey
storage pubkey: $storage_pubkey
ledger: $ledger_config_dir
======================================================================
EOF
else
if [[ $node_type = bootstrap_leader && ! -d "$SOLANA_RSYNC_CONFIG_DIR"/ledger ]]; then
ledger_not_setup "$SOLANA_RSYNC_CONFIG_DIR/ledger does not exist" ledger_not_setup "$SOLANA_RSYNC_CONFIG_DIR/ledger does not exist"
elif [[ $node_type = validator ]]; then
(
SECONDS=0
set -x
cd "$SOLANA_RSYNC_CONFIG_DIR"
$rsync -qPr "${rsync_entrypoint_url:?}"/config/ledger .
$rsync -qP "${rsync_entrypoint_url:?}"/config/state.tgz .
echo "Fetched snapshot in $SECONDS seconds"
) || true
fi fi
fi
(
set -x
if [[ $node_type = validator ]]; then
if [[ -f "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz ]]; then
mkdir -p "$state_dir"
SECONDS=
tar -C "$state_dir" -zxf "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz
echo "Extracted snapshot in $SECONDS seconds"
fi
fi
if [[ ! -d "$ledger_config_dir" ]]; then if [[ ! -d "$ledger_config_dir" ]]; then
cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger/ "$ledger_config_dir" if [[ $node_type = validator ]]; then
fi (
) cd "$SOLANA_RSYNC_CONFIG_DIR"
if ((stake_lamports)); then echo "Rsyncing genesis ledger from ${rsync_entrypoint_url:?}..."
if [[ $node_type = validator ]]; then SECONDS=
while ! $rsync -Pr "${rsync_entrypoint_url:?}"/config/ledger .; do
echo "Genesis ledger rsync failed"
sleep 5
done
echo "Fetched genesis ledger in $SECONDS seconds"
SECONDS=
echo "Rsyncing state snapshot ${rsync_entrypoint_url:?}..."
if ! $rsync -P "${rsync_entrypoint_url:?}"/config/state.tgz .; then
echo "State snapshot rsync failed"
rm -f "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz
exit
fi
echo "Fetched snapshot in $SECONDS seconds"
SECONDS=
mkdir -p "$state_dir"
(
set -x
tar -C "$state_dir" -zxf "$SOLANA_RSYNC_CONFIG_DIR"/state.tgz
)
echo "Extracted snapshot in $SECONDS seconds"
)
fi
(
set -x
cp -a "$SOLANA_RSYNC_CONFIG_DIR"/ledger/ "$ledger_config_dir"
)
fi
vote_pubkey=$($solana_keygen pubkey "$vote_keypair_path")
stake_pubkey=$($solana_keygen pubkey "$stake_keypair_path")
storage_pubkey=$($solana_keygen pubkey "$storage_keypair_path")
replace_arg --vote-account "$vote_pubkey"
if [[ $node_type = validator ]] && ((stake_lamports)); then
setup_validator_accounts "${entrypoint_address%:*}" \ setup_validator_accounts "${entrypoint_address%:*}" \
"$identity_keypair_path" \
"$vote_keypair_path" \
"$stake_keypair_path" \
"$storage_keypair_path" \
"$node_lamports" \ "$node_lamports" \
"$stake_lamports" "$stake_lamports"
elif [[ $node_type = replicator ]]; then
setup_replicator_account "${entrypoint_address%:*}" \
"$identity_keypair_path" \
"$storage_keypair_path" \
"$node_lamports"
fi fi
cat <<EOF
======================[ $node_type configuration ]======================
identity pubkey: $identity_pubkey
vote pubkey: $vote_pubkey
storage pubkey: $storage_pubkey
ledger: $ledger_config_dir
accounts: $accounts_config_dir
snapshots: $snapshot_config_dir
========================================================================
EOF
fi fi
echo "$PS4$program ${args[*]}" echo "$PS4$program ${args[*]}"
$program "${args[@]}" & $program "${args[@]}" &
pid=$! pid=$!
echo "pid: $pid" echo "pid: $pid"
@ -524,12 +522,6 @@ while true; do
if ((poll_for_new_genesis_block && --secs_to_next_genesis_poll == 0)); then if ((poll_for_new_genesis_block && --secs_to_next_genesis_poll == 0)); then
echo "Polling for new genesis block..." echo "Polling for new genesis block..."
(
set -x
$rsync -r "${rsync_entrypoint_url:?}"/config/ledger "$SOLANA_RSYNC_CONFIG_DIR"
) || (
echo "Error: failed to rsync ledger"
)
if new_gensis_block; then if new_gensis_block; then
echo "############## New genesis detected, restarting $node_type ##############" echo "############## New genesis detected, restarting $node_type ##############"
break break