Avoid |wait| as it masks failures
This commit is contained in:
@ -49,9 +49,23 @@ while read -r vmName vmZone status; do
|
|||||||
vmlist+=("$vmName:$vmZone")
|
vmlist+=("$vmName:$vmZone")
|
||||||
done < <(gcloud compute instances list --filter="$filter" --format 'value(name,zone,status)')
|
done < <(gcloud compute instances list --filter="$filter" --format 'value(name,zone,status)')
|
||||||
|
|
||||||
|
|
||||||
|
wait_for_node() {
|
||||||
|
declare pid=$1
|
||||||
|
|
||||||
|
declare ok=true
|
||||||
|
wait "$pid" || ok=false
|
||||||
|
cat "log-$pid.txt"
|
||||||
|
if ! $ok; then
|
||||||
|
echo ^^^ +++
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
echo "--- Refreshing leader for $publicUrl"
|
echo "--- Refreshing leader for $publicUrl"
|
||||||
leader=true
|
leader=true
|
||||||
logfiles=()
|
pids=()
|
||||||
for info in "${vmlist[@]}"; do
|
for info in "${vmlist[@]}"; do
|
||||||
vmName=${info%:*}
|
vmName=${info%:*}
|
||||||
vmZone=${info#*:}
|
vmZone=${info#*:}
|
||||||
@ -87,30 +101,30 @@ EOF
|
|||||||
--command="bash ./autogen-refresh-$vmName.sh"
|
--command="bash ./autogen-refresh-$vmName.sh"
|
||||||
echo "Succeeded in ${SECONDS} seconds"
|
echo "Succeeded in ${SECONDS} seconds"
|
||||||
) > "log-$vmName.txt" 2>&1 &
|
) > "log-$vmName.txt" 2>&1 &
|
||||||
|
pid=$!
|
||||||
|
# Rename log file so it can be discovered later by $pid
|
||||||
|
mv "log-$vmName.txt" "log-$pid.txt"
|
||||||
|
|
||||||
if $leader; then
|
if $leader; then
|
||||||
echo Waiting for leader...
|
echo Waiting for leader...
|
||||||
# Wait for the leader to initialize before starting the validators
|
# Wait for the leader to initialize before starting the validators
|
||||||
# TODO: Remove this limitation eventually.
|
# TODO: Remove this limitation eventually.
|
||||||
wait
|
wait_for_node "$pid"
|
||||||
|
|
||||||
cat "log-$vmName.txt"
|
|
||||||
echo "--- Refreshing validators"
|
echo "--- Refreshing validators"
|
||||||
else
|
else
|
||||||
# Slow down deployment to ~30 machines a minute to avoid triggering GCP login
|
# Slow down deployment to ~30 machines a minute to avoid triggering GCP login
|
||||||
# quota limits (the previous |scp| and |ssh| each count as a login)
|
# quota limits (the previous |scp| and |ssh| each count as a login)
|
||||||
sleep 2
|
sleep 3
|
||||||
|
|
||||||
logfiles+=("log-$vmName.txt")
|
pids+=("$pid")
|
||||||
fi
|
fi
|
||||||
leader=false
|
leader=false
|
||||||
done
|
done
|
||||||
|
|
||||||
echo --- Waiting for validators
|
echo --- Waiting for validators
|
||||||
wait
|
for pid in "${pids[@]}"; do
|
||||||
|
wait_for_node "$pid"
|
||||||
for log in "${logfiles[@]}"; do
|
|
||||||
cat "$log"
|
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "--- $publicUrl sanity test"
|
echo "--- $publicUrl sanity test"
|
||||||
|
Reference in New Issue
Block a user