Automated test framework can run scripts on launched clusters. Add offline stake operations test case and script. (#8510)

automerge

system-test/automation_utils.sh

@@ -62,6 +62,10 @@ function analyze_packet_loss {
 
 function wait_for_bootstrap_validator_stake_drop {
   max_stake="$1"
+  if [[ $max_stake -eq 100 ]]; then
+    return
+  fi
+
   source "${REPO_ROOT}"/net/common.sh
   loadConfigFile
 
@@ -85,6 +89,68 @@ function get_slot {
   ssh "${sshOptions[@]}" "${validatorIpList[0]}" '$HOME/.cargo/bin/solana slot'
 }
 
+function get_bootstrap_validator_ip_address {
+  source "${REPO_ROOT}"/net/common.sh
+  loadConfigFile
+  echo "${validatorIpList[0]}"
+}
+
+function collect_performance_statistics {
+  execution_step "Collect performance statistics about run"
+  declare q_mean_tps='
+    SELECT ROUND(MEAN("median_sum")) as "mean_tps" FROM (
+      SELECT MEDIAN(sum_count) AS "median_sum" FROM (
+        SELECT SUM("count") AS "sum_count"
+          FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
+          WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
+          GROUP BY time(1s), host_id)
+      GROUP BY time(1s)
+    )'
+
+  declare q_max_tps='
+    SELECT MAX("median_sum") as "max_tps" FROM (
+      SELECT MEDIAN(sum_count) AS "median_sum" FROM (
+        SELECT SUM("count") AS "sum_count"
+          FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
+          WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
+          GROUP BY time(1s), host_id)
+      GROUP BY time(1s)
+    )'
+
+  declare q_mean_confirmation='
+    SELECT round(mean("duration_ms")) as "mean_confirmation_ms"
+      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
+      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
+
+  declare q_max_confirmation='
+    SELECT round(max("duration_ms")) as "max_confirmation_ms"
+      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
+      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
+
+  declare q_99th_confirmation='
+    SELECT round(percentile("duration_ms", 99)) as "99th_percentile_confirmation_ms"
+      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
+      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
+
+  declare q_max_tower_distance_observed='
+    SELECT MAX("tower_distance") as "max_tower_distance" FROM (
+      SELECT last("slot") - last("root") as "tower_distance"
+        FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
+        WHERE time > now() - '"$TEST_DURATION_SECONDS"'s
+        GROUP BY time(1s), host_id)'
+
+  declare q_last_tower_distance_observed='
+      SELECT MEAN("tower_distance") as "last_tower_distance" FROM (
+            SELECT last("slot") - last("root") as "tower_distance"
+              FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
+              GROUP BY host_id)'
+
+  curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
+    --data-urlencode "db=${TESTNET_TAG}" \
+    --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" |
+    python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
+}
+
 function upload_results_to_slack() {
   echo --- Uploading results to Slack Performance Results App
 
@@ -166,7 +232,7 @@ function upload_results_to_slack() {
 		{
 			"type": "divider"
 		},
-                {
+    {
 			"type": "section",
 			"text": {
 				"type": "mrkdwn",

system-test/deprecated-testcases/colo-cpu-only-perf.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 2
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 2
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 30000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/deprecated-testcases/colo-gpu-perf.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 2
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/deprecated-testcases/gce-gpu-perf-100-node.yml

@@ -15,5 +15,6 @@ steps:
       ALLOW_BOOT_FAILURES: "true"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/partition-testcases/colo-3-partition.yml

@@ -15,5 +15,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 30
       PARTITION_INACTIVE_DURATION: 30
       PARTITION_ITERATION_COUNT: 5
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"

system-test/partition-testcases/colo-partition-long-sanity-test.yml

@@ -15,5 +15,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 60
       PARTITION_ITERATION_COUNT: 10
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"

system-test/partition-testcases/colo-partition-once-then-stabilize.yml

@@ -15,5 +15,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 300
       PARTITION_ITERATION_COUNT: 1
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"

system-test/partition-testcases/gce-5-node-3-partition.yml

@@ -18,5 +18,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 30
       PARTITION_INACTIVE_DURATION: 30
       PARTITION_ITERATION_COUNT: 5
+      TEST_TYPE: "partition"
     agents:
       - "queue=gce-deploy"

system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml

@@ -18,5 +18,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 300
       PARTITION_ITERATION_COUNT: 5
+      TEST_TYPE: "partition"
     agents:
       - "queue=gce-deploy"

system-test/partition-testcases/gce-partition-once-then-stabilize.yml

@@ -18,5 +18,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 300
       PARTITION_ITERATION_COUNT: 1
+      TEST_TYPE: "partition"
     agents:
       - "queue=gce-deploy"

system-test/performance-testcases/aws-cpu-only-perf-10-node.yml

@@ -15,5 +15,6 @@ steps:
       TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=aws-deploy"

system-test/performance-testcases/aws-cpu-only-perf-5-node.yml

@@ -15,5 +15,6 @@ steps:
       TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=aws-deploy"

system-test/performance-testcases/azure-cpu-only-perf-5-node.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "westus"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=azure-deploy"

system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 60000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/performance-testcases/gce-cpu-only-perf-10-node.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/performance-testcases/gce-cpu-only-perf-5-node.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/performance-testcases/gce-gpu-perf-10-node.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/performance-testcases/gce-gpu-perf-25-node.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/performance-testcases/gce-gpu-perf-5-node.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/performance-testcases/gce-gpu-perf-50-node.yml

@@ -15,5 +15,6 @@ steps:
       ALLOW_BOOT_FAILURES: "true"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml

@@ -6,11 +6,12 @@ steps:
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-cpu-only"
       ENABLE_GPU: "false"
-      TEST_DURATION_SECONDS: 30
+      TEST_DURATION_SECONDS: 60
       NUMBER_OF_VALIDATOR_NODES: 1
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
-      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 99
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/sanity-testcases/colo-partition-sanity-test.yml

@@ -5,7 +5,7 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-cpu-only"
-      NUMBER_OF_VALIDATOR_NODES: 4
+      NUMBER_OF_VALIDATOR_NODES: 3
       ENABLE_GPU: "false"
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"
@@ -15,6 +15,7 @@ steps:
       PARTITION_ACTIVE_DURATION: 30
       PARTITION_INACTIVE_DURATION: 30
       PARTITION_ITERATION_COUNT: 2
-      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 66
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"

system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/stability-testcases/colo-long-duration-gpu-perf.yml

@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"

system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml

@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a"
       USE_PUBLIC_IP_ADDRESSES: "false"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"

system-test/stability-testcases/gce-stability-5-node.yml

@@ -13,5 +13,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=stability-deploy"

system-test/stake-operations-testcases/offline_stake_colo.yml

@@ -0,0 +1,17 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Running Offline Stake Operations Tests on Colo"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "colo"
+      ENABLE_GPU: "false"
+      TEST_DURATION_SECONDS: 30
+      NUMBER_OF_VALIDATOR_NODES: 1
+      NUMBER_OF_CLIENT_NODES: 0
+      ADDITIONAL_FLAGS: ""
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      SKIP_PERF_RESULTS: "true"
+      TEST_TYPE: "script"
+      CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh"
+    agents:
+      - "queue=colo-deploy"

system-test/stake-operations-testcases/offline_stake_gce.yml

@@ -0,0 +1,18 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Running Offline Stake Operations Tests on GCE"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "gce"
+      ENABLE_GPU: "false"
+      TEST_DURATION_SECONDS: 30
+      NUMBER_OF_VALIDATOR_NODES: 1
+      VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
+      NUMBER_OF_CLIENT_NODES: 0
+      ADDITIONAL_FLAGS: "--dedicated"
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      SKIP_PERF_RESULTS: "true"
+      TEST_TYPE: "script"
+      CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh"
+    agents:
+      - "queue=gce-deploy"

system-test/stake-operations-testcases/offline_stake_operations.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2086
+# shellcheck disable=SC2068
+# shellcheck disable=SC2206
+# shellcheck disable=SC2162
+# shellcheck disable=SC2178
+# shellcheck disable=SC2145
+
+# shellcheck disable=SC1090
+# shellcheck disable=SC1091
+source "$(dirname "$0")"/../automation_utils.sh
+
+set -e
+
+function get_signers_string() {
+  sign_only_output="$1"
+
+  signers=()
+  while read LINE; do
+    signers+=( --signer $LINE )
+  done <<<"$(sed -Ee $'s/^  ([a-km-zA-HJ-NP-Z1-9]{32,44}=[a-km-zA-HJ-NP-Z1-9]{64,88})$/\\1/\nt\nd' <<<"$sign_only_output")"
+
+  echo "${signers[@]}"
+}
+
+if [[ -n "$1" ]]; then
+  url="$1"
+fi
+
+if [[ -z "$url" ]]; then
+  echo Provide complete URL, ex: "$0" http://devnet.solana.com:8899
+  exit 1
+fi
+solana config set --url $url
+
+# Create a dummy keypair file with no balance for operations that require a "client keypair file" to exist even if they don't touch it
+dummy_keypair=dummy.json
+solana-keygen new -o "$dummy_keypair" --no-passphrase --force --silent
+solana config set --keypair $dummy_keypair
+
+### Offline stake account creation
+
+# The nonce account and the system account funding its creation are online
+online_nonce_account_keypair=nonce_keypair.json
+online_system_account_keypair=online_system_account_keypair.json
+
+solana-keygen new -o "$online_system_account_keypair" --no-passphrase --force --silent
+solana-keygen new -o "$online_nonce_account_keypair" --no-passphrase --force --silent
+
+online_system_account_pubkey="$(solana-keygen pubkey $online_system_account_keypair)"
+nonce_account_pubkey="$(solana-keygen pubkey $online_nonce_account_keypair)"
+
+# System account funding the stake account is offline, and the auth staker and withdrawer keypairs are offline
+offline_system_account_keypair=offline_system_account_keypair.json
+offline_staker_keypair=offline_staker_keypair.json
+offline_withdrawer_keypair=offline_withdrawer_keypair.json
+offline_custodian_keypair=offline_custodian_keypair.json
+
+solana-keygen new -o "$offline_system_account_keypair" --no-passphrase --force --silent
+solana-keygen new -o "$offline_staker_keypair" --no-passphrase --force --silent
+solana-keygen new -o "$offline_withdrawer_keypair" --no-passphrase --force --silent
+solana-keygen new -o "$offline_custodian_keypair" --no-passphrase --force --silent
+
+offline_system_account_pubkey="$(solana-keygen pubkey $offline_system_account_keypair)"
+offline_withdrawer_pubkey="$(solana-keygen pubkey $offline_withdrawer_keypair)"
+offline_staker_pubkey="$(solana-keygen pubkey $offline_staker_keypair)"
+offline_custodian_pubkey="$(solana-keygen pubkey $offline_custodian_keypair)"
+
+# Airdrop some funds to the offline account.
+solana airdrop 100 $offline_system_account_pubkey
+solana airdrop 2 $online_system_account_pubkey
+
+# Create a nonce account funded by the online account, with the authority given to the offline account
+solana create-nonce-account $online_nonce_account_keypair 1 --nonce-authority $offline_system_account_pubkey --keypair $online_system_account_keypair
+nonce="$(solana nonce $nonce_account_pubkey)"
+
+execution_step OFFLINE SYSTEM ACCOUNT BALANCE BEFORE CREATING STAKE ACCOUNTS
+(
+  set -x
+  solana balance $offline_system_account_pubkey
+)
+
+################################
+execution_step CREATE OFFLINE STAKE ACCOUNT
+################################
+
+# Create a stake account funded by the offline system account
+
+stake_account_keypair=stake_account_keypair.json
+solana-keygen new -o "$stake_account_keypair" --no-passphrase --force --silent
+stake_account_address="$(solana-keygen pubkey $stake_account_keypair)"
+
+sign_only="$(solana create-stake-account $stake_account_keypair 50 \
+  --sign-only --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+  --stake-authority $offline_staker_pubkey --withdraw-authority $offline_withdrawer_pubkey \
+  --custodian $offline_custodian_pubkey \
+  --lockup-epoch 999 \
+  --keypair $offline_system_account_keypair --url http://0.0.0.0)"
+
+signers="$(get_signers_string "${sign_only[@]})")"
+
+solana create-stake-account $stake_account_keypair 50 \
+  --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
+  --stake-authority $offline_staker_pubkey --withdraw-authority $offline_withdrawer_pubkey \
+  --custodian $offline_custodian_pubkey \
+  --lockup-epoch 999 \
+  --from $offline_system_account_pubkey --fee-payer $offline_system_account_pubkey ${signers[@]}
+
+execution_step VIEW STAKE ACCOUNT AFTER CREATION
+(
+  set -x
+  solana stake-account $stake_account_address
+)
+
+
+execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER CREATING FIRST STAKE ACCOUNT
+(
+  set -x
+  solana balance $offline_system_account_pubkey
+)
+
+#####################
+execution_step SPLIT STAKE OFFLINE
+#####################
+
+# Split the original stake account before delegating
+
+split_stake_account_keypair=split_stake_account_keypair.json
+solana-keygen new -o $split_stake_account_keypair --no-passphrase --force --silent
+split_stake_account_address=$(solana-keygen pubkey $split_stake_account_keypair)
+
+nonce="$(solana nonce $nonce_account_pubkey)"
+
+sign_only="$(solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+  --stake-authority $offline_staker_keypair $stake_account_address $split_stake_account_keypair 10 \
+  --keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
+
+signers="$(get_signers_string "${sign_only[@]})")"
+
+solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
+  --stake-authority $offline_staker_pubkey $stake_account_address $split_stake_account_keypair 10 \
+  --fee-payer $offline_system_account_pubkey ${signers[@]}
+
+execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER SPLITTING
+(
+  set -x
+  solana stake-account $stake_account_address
+)
+
+execution_step VIEW NEW STAKE ACCOUNT CREATED FROM SPLITTING ORIGINAL
+(
+  set -x
+  solana stake-account $split_stake_account_address
+)
+
+#####################
+execution_step CHANGE CUSTODIAN LOCKUP
+#####################
+
+# Set the lockup epoch to 0 to allow stake to be withdrawn
+
+nonce="$(solana nonce $nonce_account_pubkey)"
+
+sign_only="$(solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+  $split_stake_account_address --custodian $offline_custodian_keypair --lockup-epoch 0 \
+  --keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
+
+signers="$(get_signers_string "${sign_only[@]})")"
+
+solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+  $split_stake_account_address --custodian $offline_custodian_pubkey --lockup-epoch 0 \
+  --fee-payer $offline_system_account_pubkey ${signers[@]}
+
+execution_step VIEW SPLIT STAKE ACCOUNT AFTER CHANGING LOCKUP
+(
+  set -x
+  solana stake-account $split_stake_account_address
+)
+
+##########################
+execution_step OFFLINE STAKE WITHDRAWAL
+##########################
+
+# Withdraw the lamports from the stake account that was split off and return them to the offline system account
+
+nonce="$(solana nonce $nonce_account_pubkey)"
+
+sign_only="$(solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+   $split_stake_account_address $offline_system_account_pubkey 10 \
+  --withdraw-authority $offline_withdrawer_keypair \
+  --keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
+
+signers="$(get_signers_string "${sign_only[@]})")"
+
+solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
+  $split_stake_account_address $offline_system_account_pubkey 10 \
+  --withdraw-authority $offline_withdrawer_pubkey \
+  --fee-payer $offline_system_account_pubkey ${signers[@]}
+
+execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER WITHDRAWING SPLIT STAKE
+(
+  set -x
+  solana balance $offline_system_account_pubkey
+)
+
+##########################
+execution_step OFFLINE STAKE DELEGATION
+##########################
+
+# Delegate stake from the original account to a vote account
+
+vote_account_pubkey="$(awk '{if(NR==4) print $2}'<<<"$(solana show-validators)")"
+nonce="$(solana nonce $nonce_account_pubkey)"
+
+# Sign a stake delegation, assuming the authorized staker is held offline
+sign_only="$(solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+--stake-authority $offline_staker_keypair $stake_account_address $vote_account_pubkey \
+--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
+
+signers="$(get_signers_string "${sign_only[@]})")"
+
+# Send the signed transaction on the cluster
+solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
+--stake-authority $offline_staker_pubkey $stake_account_address $vote_account_pubkey \
+--fee-payer $offline_system_account_pubkey ${signers[@]}
+
+execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DELEGATION
+(
+  set -x
+  solana stake-account $stake_account_address
+)
+
+##########################
+ execution_step OFFLINE STAKE DEACTIVATION
+##########################
+
+# Deactivate delegated stake
+
+nonce="$(solana nonce $nonce_account_pubkey)"
+
+# Sign a stake delegation, assuming the authorized staker is held offline
+sign_only="$(solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \
+--stake-authority $offline_staker_keypair $stake_account_address \
+--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)"
+
+signers="$(get_signers_string "${sign_only[@]})")"
+
+# Send the signed transaction on the cluster
+solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \
+--stake-authority $offline_staker_pubkey $stake_account_address \
+--fee-payer $offline_system_account_pubkey ${signers[@]}
+
+execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DEACTIVATION
+(
+   set -x
+   solana stake-account $stake_account_address
+)

system-test/stake-operations-testcases/stake_test_automation.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# shellcheck disable=SC1090
+# shellcheck disable=SC1091
+source "$(dirname "$0")"/../automation_utils.sh
+
+RESULT_FILE="$1"
+
+# Runs offline stake operations tests against a running cluster launched from the automation framework
+bootstrapper_ip_address="$(get_bootstrap_validator_ip_address)"
+entrypoint=http://"${bootstrapper_ip_address}":8899
+PATH="$REPO_ROOT"/solana-release/bin:$PATH "$REPO_ROOT"/system-test/stake-operations-testcases/offline_stake_operations.sh "$entrypoint"
+
+echo "Offline Stake Operations Test Succeeded" >>"$RESULT_FILE"

system-test/testnet-automation.sh

@@ -147,30 +147,36 @@ function launch_testnet() {
   SLOT_COUNT_START_SECONDS=$SECONDS
   execution_step "Marking beginning of slot rate test - Slot: $START_SLOT, Seconds: $SLOT_COUNT_START_SECONDS"
 
-  if [[ -n $TEST_DURATION_SECONDS ]]; then
+  case $TEST_TYPE in
-    execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test"
+    fixed_duration)
-    sleep "$TEST_DURATION_SECONDS"
+      execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test"
-  elif [[ "$APPLY_PARTITIONS" = "true" ]]; then
+      sleep "$TEST_DURATION_SECONDS"
-    STATS_START_SECONDS=$SECONDS
+      ;;
-    execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions"
+    partition)
-    sleep "$PARTITION_INACTIVE_DURATION"
+      STATS_START_SECONDS=$SECONDS
-    for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do
+      execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions"
-      execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT"
-      execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds"
-      "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE"
-      sleep "$PARTITION_ACTIVE_DURATION"
-
-      execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds"
-      "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup
       sleep "$PARTITION_INACTIVE_DURATION"
-    done
+      for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do
-    STATS_FINISH_SECONDS=$SECONDS
+        execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT"
-    TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS))
+        execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds"
-  else
+        "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE"
-    # We should never get here
+        sleep "$PARTITION_ACTIVE_DURATION"
-    echo Test duration and partition config not defined
+
-    exit 1
+        execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds"
-  fi
+        "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup
+        sleep "$PARTITION_INACTIVE_DURATION"
+      done
+      STATS_FINISH_SECONDS=$SECONDS
+      TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS))
+      ;;
+    script)
+      execution_step "Running custom script: ${REPO_ROOT}/${CUSTOM_SCRIPT}"
+      "$REPO_ROOT"/"$CUSTOM_SCRIPT" "$RESULT_FILE"
+      ;;
+    *)
+      echo "Error: Unsupported test type: $TEST_TYPE"
+      ;;
+  esac
 
   END_SLOT=$(get_slot)
   SLOT_COUNT_END_SECONDS=$SECONDS
@@ -179,63 +185,11 @@ function launch_testnet() {
   SLOTS_PER_SECOND="$(bc <<< "scale=3; ($END_SLOT - $START_SLOT)/($SLOT_COUNT_END_SECONDS - $SLOT_COUNT_START_SECONDS)")"
   execution_step "Average slot rate: $SLOTS_PER_SECOND slots/second over $((SLOT_COUNT_END_SECONDS - SLOT_COUNT_START_SECONDS)) seconds"
 
-  execution_step "Collect statistics about run"
+  if [[ "$SKIP_PERF_RESULTS" = "false" ]]; then
-  declare q_mean_tps='
+    collect_performance_statistics
-    SELECT ROUND(MEAN("median_sum")) as "mean_tps" FROM (
+    echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
-      SELECT MEDIAN(sum_count) AS "median_sum" FROM (
+  fi
-        SELECT SUM("count") AS "sum_count"
-          FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
-          WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
-          GROUP BY time(1s), host_id)
-      GROUP BY time(1s)
-    )'
 
-  declare q_max_tps='
-    SELECT MAX("median_sum") as "max_tps" FROM (
-      SELECT MEDIAN(sum_count) AS "median_sum" FROM (
-        SELECT SUM("count") AS "sum_count"
-          FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions"
-          WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0
-          GROUP BY time(1s), host_id)
-      GROUP BY time(1s)
-    )'
-
-  declare q_mean_confirmation='
-    SELECT round(mean("duration_ms")) as "mean_confirmation_ms"
-      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
-      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
-
-  declare q_max_confirmation='
-    SELECT round(max("duration_ms")) as "max_confirmation_ms"
-      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
-      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
-
-  declare q_99th_confirmation='
-    SELECT round(percentile("duration_ms", 99)) as "99th_percentile_confirmation_ms"
-      FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation"
-      WHERE time > now() - '"$TEST_DURATION_SECONDS"'s'
-
-  declare q_max_tower_distance_observed='
-    SELECT MAX("tower_distance") as "max_tower_distance" FROM (
-      SELECT last("slot") - last("root") as "tower_distance"
-        FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
-        WHERE time > now() - '"$TEST_DURATION_SECONDS"'s
-        GROUP BY time(1s), host_id)'
-
-  declare q_last_tower_distance_observed='
-      SELECT MEAN("tower_distance") as "last_tower_distance" FROM (
-            SELECT last("slot") - last("root") as "tower_distance"
-              FROM "'$TESTNET_TAG'"."autogen"."tower-observed"
-              GROUP BY host_id)'
-
-  curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \
-    --data-urlencode "db=${TESTNET_TAG}" \
-    --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" |
-    python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE"
-
-  echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
-
-  execution_step "Writing test results to ${RESULT_FILE}"
   RESULT_DETAILS=$(<"$RESULT_FILE")
   upload-ci-artifact "$RESULT_FILE"
 }
@@ -245,9 +199,10 @@ RESULT_DETAILS=
 STEP=
 execution_step "Initialize Environment"
 
-[[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation
+[[ -n $TESTNET_TAG ]] || TESTNET_TAG=${CLOUD_PROVIDER}-testnet-automation
 [[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086
 [[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66
+[[ -n $SKIP_PERF_RESULTS ]] || SKIP_PERF_RESULTS=false
 
 if [[ -z $NUMBER_OF_VALIDATOR_NODES ]]; then
   echo NUMBER_OF_VALIDATOR_NODES not defined
@@ -292,18 +247,41 @@ if [[ "$USE_PUBLIC_IP_ADDRESSES" = "true" ]]; then
   maybePublicIpAddresses="-P"
 fi
 
-: "${CLIENT_DELAY_START:=0}"
+execution_step "Checking for required parameters"
+testTypeRequiredParameters=
+case $TEST_TYPE in
+  fixed_duration)
+    testTypeRequiredParameters=(
+      TEST_DURATION_SECONDS \
+      )
+    ;;
+  partition)
+    testTypeRequiredParameters=(
+      NETEM_CONFIG_FILE \
+      PARTITION_ACTIVE_DURATION \
+      PARTITION_INACTIVE_DURATION \
+      PARTITION_ITERATION_COUNT \
+    )
+    ;;
+  script)
+    testTypeRequiredParameters=(
+      CUSTOM_SCRIPT \
+    )
+    ;;
+  *)
+    echo "Error: Unsupported test type: $TEST_TYPE"
+    ;;
+esac
 
-if [[ -z $APPLY_PARTITIONS ]]; then
+missingParameters=
-  APPLY_PARTITIONS=false
+for i in "${testTypeRequiredParameters[@]}"; do
-fi
+  if [[ -z ${!i} ]]; then
-if [[ "$APPLY_PARTITIONS" = "true" ]]; then
+    missingParameters+="${i}, "
-  if [[ -n $TEST_DURATION_SECONDS ]]; then
-    echo Cannot accept TEST_DURATION_SECONDS and a parition looping config
-    exit 1
   fi
-elif [[ -z $TEST_DURATION_SECONDS ]]; then
+done
-  echo TEST_DURATION_SECONDS not defined
+
+if [[ -n $missingParameters ]]; then
+  echo "Error: For test type $TEST_TYPE, the following required parameters are missing: ${missingParameters[*]}"
   exit 1
 fi
 
@@ -337,6 +315,8 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \
                         PARTITION_ACTIVE_DURATION \
                         PARTITION_INACTIVE_DURATION \
                         PARTITION_ITERATION_COUNT \
+                        TEST_TYPE \
+                        CUSTOM_SCRIPT \
                         )
 
 TEST_CONFIGURATION=