Only check CHANNEL_BRANCH if TESTNET_TAG is not set from buildkite (#3698)

This reverts commit f4f22d3f7b.

ci/testnet-deploy.sh

@@ -27,13 +27,14 @@ usage() {
     echo "Error: $*"
   fi
   cat <<EOF
-usage: $0 [name] [cloud] [zone] [options...]
+usage: $0 -p network-name -C cloud -z zone1 [-z zone2] ... [-z zoneN] [options...]
 
 Deploys a CD testnet
 
-  name  - name of the network
-  cloud - cloud provider to use (gce, ec2)
-  zone  - cloud provider zone to deploy the network into
+  mandatory arguments:
+  -p [network-name]  - name of the network
+  -C [cloud] - cloud provider to use (gce, ec2)
+  -z [zone]  - cloud provider zone to deploy the network into.  Must specify at least one zone
 
   options:
    -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
@@ -59,19 +60,22 @@ EOF
   exit $exitcode
 }
 
-netName=$1
-cloudProvider=$2
-zone=$3
-[[ -n $netName ]] || usage
-[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
-[[ -n $zone ]] || usage "Zone not specified"
-shift 3
+zone=()
 
-while getopts "h?p:Pn:c:t:gG:a:Dbd:rusx" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:" opt; do
   case $opt in
   h | \?)
     usage
     ;;
+  p)
+    netName=$OPTARG
+    ;;
+  C)
+    cloudProvider=$OPTARG
+    ;;
+  z)
+    zone+=("$OPTARG")
+    ;;
   P)
     publicNetwork=true
     ;;
@@ -128,6 +132,10 @@ while getopts "h?p:Pn:c:t:gG:a:Dbd:rusx" opt; do
   esac
 done
 
+[[ -n $netName ]] || usage
+[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
+[[ -n ${zone[*]} ]] || usage "At least one zone must be specified"
+
 shutdown() {
   exitcode=$?
 
@@ -148,9 +156,16 @@ trap shutdown EXIT INT
 
 set -x
 
+# Build a string to pass zone opts to $cloudProvider.sh: "-z zone1 -z zone2 ..."
+zone_args=()
+for val in "${zone[@]}"; do
+  zone_args+=("-z $val")
+done
+
 if ! $skipSetup; then
   echo "--- $cloudProvider.sh delete"
-  time net/"$cloudProvider".sh delete -z "$zone" -p "$netName" ${externalNode:+-x}
+  # shellcheck disable=SC2068
+  time net/"$cloudProvider".sh delete ${zone_args[@]} -p "$netName" ${externalNode:+-x}
   if $delete; then
     exit 0
   fi
@@ -158,11 +173,12 @@ if ! $skipSetup; then
   echo "--- $cloudProvider.sh create"
   create_args=(
     -p "$netName"
-    -z "$zone"
     -a "$bootstrapFullNodeAddress"
     -c "$clientNodeCount"
     -n "$additionalFullNodeCount"
   )
+  # shellcheck disable=SC2206
+  create_args+=(${zone_args[@]})
 
   if $blockstreamer; then
     create_args+=(-u)
@@ -197,8 +213,9 @@ else
   echo "--- $cloudProvider.sh config"
   config_args=(
     -p "$netName"
-    -z "$zone"
   )
+  # shellcheck disable=SC2206
+  config_args+=(${zone_args[@]})
   if $publicNetwork; then
     config_args+=(-P)
   fi

ci/testnet-manager.sh

@@ -75,28 +75,31 @@ source scripts/configure-metrics.sh
 ci/channel-info.sh
 eval "$(ci/channel-info.sh)"
 
-case $TESTNET in
-testnet-edge|testnet-edge-perf)
-  CHANNEL_OR_TAG=edge
-  CHANNEL_BRANCH=$EDGE_CHANNEL
-  ;;
-testnet-beta|testnet-beta-perf)
-  CHANNEL_OR_TAG=beta
-  CHANNEL_BRANCH=$BETA_CHANNEL
-  ;;
-testnet|testnet-perf)
-  CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
-  CHANNEL_BRANCH=$STABLE_CHANNEL
-  ;;
-*)
-  echo "Error: Invalid TESTNET=$TESTNET"
-  exit 1
-  ;;
-esac
+if [[ -n $TESTNET_TAG ]]; then
+  CHANNEL_OR_TAG=$TESTNET_TAG
+else
+  case $TESTNET in
+  testnet-edge|testnet-edge-perf)
+    CHANNEL_OR_TAG=edge
+    CHANNEL_BRANCH=$EDGE_CHANNEL
+    ;;
+  testnet-beta|testnet-beta-perf)
+    CHANNEL_OR_TAG=beta
+    CHANNEL_BRANCH=$BETA_CHANNEL
+    ;;
+  testnet|testnet-perf)
+    CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
+    CHANNEL_BRANCH=$STABLE_CHANNEL
+    ;;
+  *)
+    echo "Error: Invalid TESTNET=$TESTNET"
+    exit 1
+    ;;
+  esac
 
-if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
-  (
-    cat <<EOF
+  if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+    (
+      cat <<EOF
 steps:
   - trigger: "$BUILDKITE_PIPELINE_SLUG"
     async: true
@@ -107,9 +110,12 @@ steps:
         TESTNET: "$TESTNET"
         TESTNET_OP: "$TESTNET_OP"
         TESTNET_DB_HOST: "$TESTNET_DB_HOST"
+        EC2_NODE_COUNT: "$EC2_NODE_COUNT"
+        GCE_NODE_COUNT: "$GCE_NODE_COUNT"
 EOF
-  ) | buildkite-agent pipeline upload
-  exit 0
+    ) | buildkite-agent pipeline upload
+    exit 0
+  fi
 fi
 
 
@@ -134,7 +140,23 @@ sanity() {
   testnet-beta)
     (
       set -x
-      ci/testnet-sanity.sh beta-testnet-solana-com ec2 us-west-1a
+      EC2_ZONES=(us-west-1a sa-east-1a ap-northeast-2a eu-central-1a ca-central-1a)
+      ok=true
+      for zone in "${EC2_ZONES[@]}"; do
+        if ! $ok; then
+          break
+        fi
+        ci/testnet-sanity.sh beta-testnet-solana-com ec2 "$zone" || ok=false
+      done
+
+      GCE_ZONES=(us-west1-b asia-east2-a europe-west4-a southamerica-east1-b us-east4-c)
+      for zone in "${GCE_ZONES[@]}"; do
+        if ! $ok; then
+          break
+        fi
+        ci/testnet-sanity.sh beta-testnet-solana-com gce "$zone" || ok=false
+      done
+      $ok
     )
     ;;
   testnet-beta-perf)
@@ -186,7 +208,7 @@ start() {
       set -x
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
+        ci/testnet-deploy.sh -p edge-testnet-solana-com -C ec2 -z us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
@@ -197,7 +219,7 @@ start() {
       set -x
       NO_LEDGER_VERIFY=1 \
       NO_VALIDATOR_SANITY=1 \
-        ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
+        ci/testnet-deploy.sh -p edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           ${maybeReuseLedger:+-r} \
@@ -207,16 +229,35 @@ start() {
   testnet-beta)
     (
       set -x
+      EC2_ZONES=(us-west-1a sa-east-1a ap-northeast-2a eu-central-1a ca-central-1a)
+      GCE_ZONES=(us-west1-b asia-east2-a europe-west4-a southamerica-east1-b us-east4-c)
+
+      # Build an array to pass as opts to testnet-deploy.sh: "-z zone1 -z zone2 ..."
+      GCE_ZONE_ARGS=()
+      for val in "${GCE_ZONES[@]}"; do
+        GCE_ZONE_ARGS+=("-z $val")
+      done
+
+      EC2_ZONE_ARGS=()
+      for val in "${EC2_ZONES[@]}"; do
+        EC2_ZONE_ARGS+=("-z $val")
+      done
+
+      [[ -n $EC2_NODE_COUNT ]] || EC2_NODE_COUNT=60
+      [[ -n $GCE_NODE_COUNT ]] || GCE_NODE_COUNT=40
+
+      # shellcheck disable=SC2068
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
-          -t "$CHANNEL_OR_TAG" -n 35 -c 0 -s -u -P -a eipalloc-0f286cf8a0771ce35 \
+        ci/testnet-deploy.sh -p beta-testnet-solana-com -C ec2 ${EC2_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -s -u -P -a eipalloc-0f286cf8a0771ce35 \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
+      # shellcheck disable=SC2068
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh beta-testnet-solana-com gce us-west1-a \
-          -t "$CHANNEL_OR_TAG" -n 65 -c 0 -x -P \
+        ci/testnet-deploy.sh -p beta-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -x -P \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
     )
@@ -226,7 +267,7 @@ start() {
       set -x
       NO_LEDGER_VERIFY=1 \
       NO_VALIDATOR_SANITY=1 \
-        ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
+        ci/testnet-deploy.sh -p beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           ${maybeReuseLedger:+-r} \
@@ -238,12 +279,12 @@ start() {
       set -x
       NO_VALIDATOR_SANITY=1 \
       RUST_LOG=solana=info \
-        ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
+        ci/testnet-deploy.sh -p testnet-solana-com -C ec2 -z us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0fa502bf95f6f18b2 \
           -b \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
-        #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
+        #ci/testnet-deploy.sh -p testnet-solana-com -C gce -z us-east1-c \
         #  -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
         #  ${maybeReuseLedger:+-r} \
         #  ${maybeDelete:+-D}
@@ -254,14 +295,14 @@ start() {
       set -x
       NO_LEDGER_VERIFY=1 \
       NO_VALIDATOR_SANITY=1 \
-        ci/testnet-deploy.sh perf-testnet-solana-com gce us-west1-b \
+        ci/testnet-deploy.sh -p perf-testnet-solana-com -C gce -z us-west1-b \
           -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
           -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           -d pd-ssd \
           ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-D}
-        #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
+        #ci/testnet-deploy.sh -p perf-testnet-solana-com -C ec2 -z us-east-1a \
         #  -g \
         #  -t "$CHANNEL_OR_TAG" -c 2 \
         #  ${maybeReuseLedger:+-r} \

ci/testnet-sanity.sh

@@ -48,6 +48,7 @@ shutdown() {
   exit $exitcode
 }
 rm -rf net/{log,-sanity}
+rm -f net/config/config
 trap shutdown EXIT INT
 
 set -x

core/src/locktower.rs

@@ -171,8 +171,24 @@ impl Locktower {
                 };
                 Self::update_ancestor_lockouts(&mut stake_lockouts, &vote, ancestors);
             }
-            // each account hash a stake for all the forks in the active tree for this bank
-            Self::update_ancestor_stakes(&mut stake_lockouts, bank_slot, lamports, ancestors);
+
+            // The last vote in the vote stack is a simulated vote on bank_slot, which
+            // we added to the vote stack earlier in this function by calling process_vote().
+            // We don't want to update the ancestors stakes of this vote b/c it does not
+            // represent an actual vote by the validator.
+
+            // Note: It should not be possible for any vote state in this bank to have
+            // a vote for a slot >= bank_slot, so we are guaranteed that the last vote in
+            // this vote stack is the simulated vote, so this fetch should be sufficient
+            // to find the last unsimulated vote.
+            assert_eq!(
+                vote_state.nth_recent_vote(0).map(|l| l.slot),
+                Some(bank_slot)
+            );
+            if let Some(vote) = vote_state.nth_recent_vote(1) {
+                // Update all the parents of this last vote with the stake of this vote account
+                Self::update_ancestor_stakes(&mut stake_lockouts, vote.slot, lamports, ancestors);
+            }
         }
         stake_lockouts
     }
@@ -690,4 +706,55 @@ mod test {
         assert_eq!(stake_lockouts[&1].stake, 1);
         assert_eq!(stake_lockouts[&2].stake, 1);
     }
+
+    #[test]
+    fn test_check_vote_threshold_forks() {
+        // Create the ancestor relationships
+        let ancestors = (0..=(VOTE_THRESHOLD_DEPTH + 1) as u64)
+            .map(|slot| {
+                let slot_parents: HashSet<_> = (0..slot).collect();
+                (slot, slot_parents)
+            })
+            .collect();
+
+        // Create votes such that
+        // 1) 3/4 of the stake has voted on slot: VOTE_THRESHOLD_DEPTH - 2, lockout: 2
+        // 2) 1/4 of the stake has voted on slot: VOTE_THRESHOLD_DEPTH, lockout: 2^9
+        let total_stake = 4;
+        let threshold_size = 0.67;
+        let threshold_stake = (f64::ceil(total_stake as f64 * threshold_size)) as u64;
+        let locktower_votes: Vec<u64> = (0..VOTE_THRESHOLD_DEPTH as u64).collect();
+        let accounts = gen_accounts(&[
+            (threshold_stake, &[(VOTE_THRESHOLD_DEPTH - 2) as u64]),
+            (total_stake - threshold_stake, &locktower_votes[..]),
+        ]);
+
+        // Initialize locktower
+        let stakes: HashMap<_, _> = accounts.iter().map(|(pk, a)| (*pk, a.lamports)).collect();
+        let epoch_stakes = EpochStakes::new(0, stakes, &Pubkey::default());
+        let mut locktower = Locktower::new(epoch_stakes, VOTE_THRESHOLD_DEPTH, threshold_size);
+
+        // CASE 1: Record the first VOTE_THRESHOLD locktower votes for fork 2. We want to
+        // evaluate a vote on slot VOTE_THRESHOLD_DEPTH. The nth most recent vote should be
+        // for slot 0, which is common to all account vote states, so we should pass the
+        // threshold check
+        let vote_to_evaluate = VOTE_THRESHOLD_DEPTH as u64;
+        for vote in &locktower_votes {
+            locktower.record_vote(*vote);
+        }
+        let stakes_lockouts = locktower.collect_vote_lockouts(
+            vote_to_evaluate,
+            accounts.clone().into_iter(),
+            &ancestors,
+        );
+        assert!(locktower.check_vote_stake_threshold(vote_to_evaluate, &stakes_lockouts));
+
+        // CASE 2: Now we want to evaluate a vote for slot VOTE_THRESHOLD_DEPTH + 1. This slot
+        // will expire the vote in one of the vote accounts, so we should have insufficient
+        // stake to pass the threshold
+        let vote_to_evaluate = VOTE_THRESHOLD_DEPTH as u64 + 1;
+        let stakes_lockouts =
+            locktower.collect_vote_lockouts(vote_to_evaluate, accounts.into_iter(), &ancestors);
+        assert!(!locktower.check_vote_stake_threshold(vote_to_evaluate, &stakes_lockouts));
+    }
 }

net/gce.sh

@@ -23,7 +23,7 @@ ec2)
   # shellcheck source=net/scripts/ec2-provider.sh
   source "$here"/scripts/ec2-provider.sh
 
-  cpuBootstrapLeaderMachineType=m4.4xlarge
+  cpuBootstrapLeaderMachineType=m4.2xlarge
   gpuBootstrapLeaderMachineType=p2.xlarge
   bootstrapLeaderMachineType=$cpuBootstrapLeaderMachineType
   fullNodeMachineType=$cpuBootstrapLeaderMachineType
@@ -517,9 +517,11 @@ EOF
     fi
     cloud_CreateInstances "$prefix" "$prefix-$zone-fullnode" "$numNodesPerZone" \
       "$enableGpu" "$fullNodeMachineType" "$zone" "$fullNodeBootDiskSizeInGb" \
-      "$startupScript" "" "$bootDiskType"
+      "$startupScript" "" "$bootDiskType" &
   done
 
+  wait
+
   if [[ $clientNodeCount -gt 0 ]]; then
     cloud_CreateInstances "$prefix" "$prefix-client" "$clientNodeCount" \
       "$enableGpu" "$clientMachineType" "${zones[0]}" "$clientBootDiskSizeInGb" \

net/scripts/ec2-provider.sh

@@ -49,7 +49,7 @@ __cloud_FindInstances() {
   declare filter="$1"
 
   instances=()
-  declare -a regions=("us-east-1" "us-west-1" "us-west-2")
+  declare -a regions=("us-east-1" "us-west-1" "us-west-2" "sa-east-1" "ap-northeast-2" "eu-central-1" "ca-central-1")
   for region in "${regions[@]}"
   do
     declare name publicIp privateIp
@@ -202,6 +202,18 @@ cloud_CreateInstances() {
     us-west-2)
       imageName="ami-0dc34f4b016c9ce49"
       ;;
+    sa-east-1)
+      imageName="ami-0f1678b6f63a0f923"
+      ;;
+    ap-northeast-2)
+      imageName="ami-0695e34e31339c3ff"
+      ;;
+    eu-central-1)
+      imageName="ami-054e21e355db24124"
+      ;;
+    ca-central-1)
+      imageName="ami-06ed08059bdc08fc9"
+      ;;
     *)
       usage "Unsupported region: $region"
       ;;