testnet-demo now runs across more GCE zones (#4053)
* testnet-demo now runs across more GCE zones * Save zone info to config file * Add geoip whitelist for common data centers * Skip more of start * Include -x for config * Fetch private key from first validator node if necessary * Correct -r propagation
This commit is contained in:
Michael Vines
134
net/gce.sh
134
net/gce.sh
@ -227,6 +227,7 @@ esac
|
||||
# name - name of the instance
|
||||
# publicIp - The public IP address of this instance
|
||||
# privateIp - The private IP address of this instance
|
||||
# zone - Zone of this instance
|
||||
# count - Monotonically increasing count for each
|
||||
# invocation of cmd, starting at 1
|
||||
# ... - Extra args to cmd..
|
||||
@ -242,11 +243,70 @@ cloud_ForEachInstance() {
|
||||
declare name publicIp privateIp
|
||||
IFS=: read -r name publicIp privateIp zone < <(echo "$info")
|
||||
|
||||
eval "$cmd" "$name" "$publicIp" "$privateIp" "$count" "$@"
|
||||
eval "$cmd" "$name" "$publicIp" "$privateIp" "$zone" "$count" "$@"
|
||||
count=$((count + 1))
|
||||
done
|
||||
}
|
||||
|
||||
# Given a cloud provider zone, return an approximate lat,long location for the
|
||||
# data center. Normal geoip lookups for cloud provider IP addresses are
|
||||
# sometimes widely inaccurate.
|
||||
zoneLocation() {
|
||||
declare zone="$1"
|
||||
case "$zone" in
|
||||
us-west1*)
|
||||
echo "[45.5946, -121.1787]"
|
||||
;;
|
||||
us-central1*)
|
||||
echo "[41.2619, -95.8608]"
|
||||
;;
|
||||
us-east1*)
|
||||
echo "[33.1960, -80.0131]"
|
||||
;;
|
||||
asia-east2*)
|
||||
echo "[22.3193, 114.1694]"
|
||||
;;
|
||||
asia-northeast1*)
|
||||
echo "[35.6762, 139.6503]"
|
||||
;;
|
||||
asia-northeast2*)
|
||||
echo "[34.6937, 135.5023]"
|
||||
;;
|
||||
asia-south1*)
|
||||
echo "[19.0760, 72.8777]"
|
||||
;;
|
||||
asia-southeast1*)
|
||||
echo "[1.3404, 103.7090]"
|
||||
;;
|
||||
australia-southeast1*)
|
||||
echo "[-33.8688, 151.2093]"
|
||||
;;
|
||||
europe-north1*)
|
||||
echo "[60.5693, 27.1878]"
|
||||
;;
|
||||
europe-west2*)
|
||||
echo "[51.5074, -0.1278]"
|
||||
;;
|
||||
europe-west3*)
|
||||
echo "[50.1109, 8.6821]"
|
||||
;;
|
||||
europe-west4*)
|
||||
echo "[53.4386, 6.8355]"
|
||||
;;
|
||||
europe-west6*)
|
||||
echo "[47.3769, 8.5417]"
|
||||
;;
|
||||
northamerica-northeast1*)
|
||||
echo "[45.5017, -73.5673]"
|
||||
;;
|
||||
southamerica-east1*)
|
||||
echo "[-23.5505, -46.6333]"
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
prepareInstancesAndWriteConfigFile() {
|
||||
$metricsWriteDatapoint "testnet-deploy net-config-begin=1"
|
||||
|
||||
@ -254,6 +314,7 @@ prepareInstancesAndWriteConfigFile() {
|
||||
echo "Appending to existing config file"
|
||||
echo "externalNodeSshKey=$sshPrivateKey" >> "$configFile"
|
||||
else
|
||||
rm -f "$geoipConfigFile"
|
||||
cat >> "$configFile" <<EOF
|
||||
# autogenerated at $(date)
|
||||
netBasename=$prefix
|
||||
@ -262,6 +323,7 @@ sshPrivateKey=$sshPrivateKey
|
||||
leaderRotation=$leaderRotation
|
||||
EOF
|
||||
fi
|
||||
touch "$geoipConfigFile"
|
||||
|
||||
buildSshOptions
|
||||
|
||||
@ -269,11 +331,13 @@ EOF
|
||||
declare name="$1"
|
||||
declare publicIp="$2"
|
||||
declare privateIp="$3"
|
||||
declare zone="$4"
|
||||
#declare index="$5"
|
||||
|
||||
declare failOnFailure="$5"
|
||||
declare arrayName="$6"
|
||||
declare failOnFailure="$6"
|
||||
declare arrayName="$7"
|
||||
|
||||
# This check should eventually be moved to cloud provider specific script
|
||||
# This check should eventually be moved to cloud provider specific script
|
||||
if [ "$publicIp" = "TERMINATED" ] || [ "$privateIp" = "TERMINATED" ]; then
|
||||
if $failOnFailure; then
|
||||
exit 1
|
||||
@ -286,7 +350,7 @@ EOF
|
||||
echo "Waiting for $name to finish booting..."
|
||||
(
|
||||
set -x +e
|
||||
for i in $(seq 1 20); do
|
||||
for i in $(seq 1 30); do
|
||||
timeout --preserve-status --foreground 20s ssh "${sshOptions[@]}" "$publicIp" "ls -l /.instance-startup-complete"
|
||||
ret=$?
|
||||
if [[ $ret -eq 0 ]]; then
|
||||
@ -305,21 +369,21 @@ EOF
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "$arrayName+=($publicIp) # $name" >> "$configFile"
|
||||
echo "${arrayName}Private+=($privateIp) # $name" >> "$configFile"
|
||||
{
|
||||
echo "$arrayName+=($publicIp) # $name"
|
||||
echo "${arrayName}Private+=($privateIp) # $name"
|
||||
echo "${arrayName}Zone+=($zone) # $name"
|
||||
} >> "$configFile"
|
||||
|
||||
declare latlng=
|
||||
latlng=$(zoneLocation "$zone")
|
||||
if [[ -n $latlng ]]; then
|
||||
echo "$publicIp: $latlng" >> "$geoipConfigFile"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if $externalNodes; then
|
||||
echo "Bootstrap leader is already configured"
|
||||
else
|
||||
echo "Looking for bootstrap leader instance..."
|
||||
cloud_FindInstance "$prefix-bootstrap-leader"
|
||||
[[ ${#instances[@]} -eq 1 ]] || {
|
||||
echo "Unable to find bootstrap leader"
|
||||
exit 1
|
||||
}
|
||||
|
||||
fetchPrivateKey() {
|
||||
(
|
||||
declare nodeName
|
||||
declare nodeIp
|
||||
@ -338,7 +402,9 @@ EOF
|
||||
set -x -o pipefail
|
||||
for i in $(seq 1 30); do
|
||||
if cloud_FetchFile "$nodeName" "$nodeIp" /solana-id_ecdsa "$sshPrivateKey" "$nodeZone"; then
|
||||
break
|
||||
if cloud_FetchFile "$nodeName" "$nodeIp" /solana-id_ecdsa.pub "$sshPrivateKey.pub" "$nodeZone"; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
@ -350,6 +416,20 @@ EOF
|
||||
fi
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
if $externalNodes; then
|
||||
echo "Bootstrap leader is already configured"
|
||||
else
|
||||
echo "Looking for bootstrap leader instance..."
|
||||
cloud_FindInstance "$prefix-bootstrap-leader"
|
||||
[[ ${#instances[@]} -eq 1 ]] || {
|
||||
echo "Unable to find bootstrap leader"
|
||||
exit 1
|
||||
}
|
||||
|
||||
fetchPrivateKey
|
||||
|
||||
echo "fullnodeIpList=()" >> "$configFile"
|
||||
echo "fullnodeIpListPrivate=()" >> "$configFile"
|
||||
cloud_ForEachInstance recordInstanceIp true fullnodeIpList
|
||||
@ -363,6 +443,8 @@ EOF
|
||||
echo "Unable to find additional fullnodes"
|
||||
exit 1
|
||||
}
|
||||
|
||||
fetchPrivateKey
|
||||
cloud_ForEachInstance recordInstanceIp "$failOnValidatorBootupFailure" fullnodeIpList
|
||||
done
|
||||
fi
|
||||
@ -588,29 +670,33 @@ info)
|
||||
declare nodeType=$1
|
||||
declare ip=$2
|
||||
declare ipPrivate=$3
|
||||
printf " %-16s | %-15s | %-15s\n" "$nodeType" "$ip" "$ipPrivate"
|
||||
declare zone=$4
|
||||
printf " %-16s | %-15s | %-15s | %s\n" "$nodeType" "$ip" "$ipPrivate" "$zone"
|
||||
}
|
||||
|
||||
printNode "Node Type" "Public IP" "Private IP"
|
||||
echo "-------------------+-----------------+-----------------"
|
||||
printNode "Node Type" "Public IP" "Private IP" "Zone"
|
||||
echo "-------------------+-----------------+-----------------+--------------"
|
||||
nodeType=bootstrap-leader
|
||||
for i in $(seq 0 $(( ${#fullnodeIpList[@]} - 1)) ); do
|
||||
ipAddress=${fullnodeIpList[$i]}
|
||||
ipAddressPrivate=${fullnodeIpListPrivate[$i]}
|
||||
printNode $nodeType "$ipAddress" "$ipAddressPrivate"
|
||||
zone=${fullnodeIpListZone[$i]}
|
||||
printNode $nodeType "$ipAddress" "$ipAddressPrivate" "$zone"
|
||||
nodeType=fullnode
|
||||
done
|
||||
|
||||
for i in $(seq 0 $(( ${#clientIpList[@]} - 1)) ); do
|
||||
ipAddress=${clientIpList[$i]}
|
||||
ipAddressPrivate=${clientIpListPrivate[$i]}
|
||||
printNode bench-tps "$ipAddress" "$ipAddressPrivate"
|
||||
zone=${clientIpListZone[$i]}
|
||||
printNode bench-tps "$ipAddress" "$ipAddressPrivate" "$zone"
|
||||
done
|
||||
|
||||
for i in $(seq 0 $(( ${#blockstreamerIpList[@]} - 1)) ); do
|
||||
ipAddress=${blockstreamerIpList[$i]}
|
||||
ipAddressPrivate=${blockstreamerIpListPrivate[$i]}
|
||||
printNode blockstreamer "$ipAddress" "$ipAddressPrivate"
|
||||
zone=${blockstreamerIpListZone[$i]}
|
||||
printNode blockstreamer "$ipAddress" "$ipAddressPrivate" "$zone"
|
||||
done
|
||||
;;
|
||||
*)
|
||||
|
||||
Reference in New Issue
Block a user