Add system test to measure recovery after partition (#20902)

* Add system test to measure recovery after partition

* shellcheck

* increase partition length until failure

* adjust parameters and output

* different stopping condition

system-test/automation_utils.sh

@@ -110,6 +110,21 @@ function get_current_stake {
     '$HOME/.cargo/bin/solana --url http://127.0.0.1:8899 validators --output=json | grep -o "totalCurrentStake\": [0-9]*" | cut -d: -f2'
 }
 
+function get_validator_confirmation_time {
+  SINCE=$1
+  declare q_mean_confirmation='
+    SELECT ROUND(MEAN("duration_ms")) as "mean_confirmation_ms"
+      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
+      WHERE time > now() - '"$SINCE"'s'
+
+  mean_confirmation_ms=$( \
+      curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
+        --data-urlencode "db=${TESTNET_TAG}" \
+        --data-urlencode "q=$q_mean_confirmation" |
+      python3 "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py --empty_error |
+      cut -d' ' -f2)
+}
+
 function collect_performance_statistics {
   execution_step "Collect performance statistics about run"
   declare q_mean_tps='