Facility to provision primordial accounts for fullnodes in genesis block (#4631)

* updated usage

* shellcheck

* support replicators

* disable airdrops if primordial accounts are used

* review comments

genesis/src/main.rs

@@ -13,7 +13,6 @@ extern crate solana_config_program;
 extern crate solana_exchange_program;
 
 use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg};
-use serde_derive::{Deserialize, Serialize};
 use solana::blocktree::create_new_ledger;
 use solana_sdk::account::Account;
 use solana_sdk::fee_calculator::FeeCalculator;
@@ -27,29 +26,25 @@ use solana_sdk::timing;
 use solana_stake_api::stake_state;
 use solana_storage_program::genesis_block_util::GenesisBlockUtil;
 use solana_vote_api::vote_state;
+use std::collections::HashMap;
 use std::error;
 use std::fs::File;
 use std::io;
+use std::str::FromStr;
 use std::time::{Duration, Instant};
 
 pub const BOOTSTRAP_LEADER_LAMPORTS: u64 = 42;
 
-#[derive(Serialize, Deserialize, Default, Debug, PartialEq)]
-pub struct PrimordialAccount {
-    pub pubkey: Pubkey,
-    pub lamports: u64,
-}
-
 pub fn append_primordial_accounts(file: &str, genesis_block: &mut GenesisBlock) -> io::Result<()> {
     let accounts_file = File::open(file.to_string())?;
 
-    let primordial_accounts: Vec<PrimordialAccount> = serde_yaml::from_reader(accounts_file)
+    let primordial_accounts: HashMap<String, u64> = serde_yaml::from_reader(accounts_file)
         .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
 
-    primordial_accounts.iter().for_each(|primordial| {
+    primordial_accounts.into_iter().for_each(|primordial| {
         genesis_block.accounts.push((
-            primordial.pubkey,
+            Pubkey::from_str(primordial.0.as_str()).unwrap(),
-            Account::new(primordial.lamports, 0, &system_program::id()),
+            Account::new(primordial.1, 0, &system_program::id()),
         ))
     });
 
@@ -314,6 +309,7 @@ mod tests {
     use hashbrown::HashSet;
     use solana_sdk::genesis_block::GenesisBlock;
     use solana_sdk::pubkey::Pubkey;
+    use std::collections::HashMap;
     use std::fs::remove_file;
     use std::io::Write;
     use std::path::Path;
@@ -343,20 +339,10 @@ mod tests {
         // Test invalid file returns error
         assert!(append_primordial_accounts("unknownfile", &mut genesis_block).is_err());
 
-        let primordial_accounts = [
+        let mut primordial_accounts = HashMap::new();
-            PrimordialAccount {
+        primordial_accounts.insert(Pubkey::new_rand().to_string(), 2 as u64);
-                pubkey: Pubkey::new_rand(),
+        primordial_accounts.insert(Pubkey::new_rand().to_string(), 1 as u64);
-                lamports: 2,
+        primordial_accounts.insert(Pubkey::new_rand().to_string(), 3 as u64);
-            },
-            PrimordialAccount {
-                pubkey: Pubkey::new_rand(),
-                lamports: 1,
-            },
-            PrimordialAccount {
-                pubkey: Pubkey::new_rand(),
-                lamports: 3,
-            },
-        ];
 
         let serialized = serde_yaml::to_string(&primordial_accounts).unwrap();
         let path = Path::new("test_append_primordial_accounts_to_genesis.yml");
@@ -377,28 +363,17 @@ mod tests {
 
         // Test account data matches
         (0..primordial_accounts.len()).for_each(|i| {
-            assert_eq!(genesis_block.accounts[i].0, primordial_accounts[i].pubkey);
             assert_eq!(
+                primordial_accounts[&genesis_block.accounts[i].0.to_string()],
                 genesis_block.accounts[i].1.lamports,
-                primordial_accounts[i].lamports
             );
         });
 
         // Test more accounts can be appended
-        let primordial_accounts1 = [
+        let mut primordial_accounts1 = HashMap::new();
-            PrimordialAccount {
+        primordial_accounts1.insert(Pubkey::new_rand().to_string(), 6 as u64);
-                pubkey: Pubkey::new_rand(),
+        primordial_accounts1.insert(Pubkey::new_rand().to_string(), 5 as u64);
-                lamports: 6,
+        primordial_accounts1.insert(Pubkey::new_rand().to_string(), 10 as u64);
-            },
-            PrimordialAccount {
-                pubkey: Pubkey::new_rand(),
-                lamports: 5,
-            },
-            PrimordialAccount {
-                pubkey: Pubkey::new_rand(),
-                lamports: 10,
-            },
-        ];
 
         let serialized = serde_yaml::to_string(&primordial_accounts1).unwrap();
         let path = Path::new("test_append_primordial_accounts_to_genesis.yml");
@@ -421,24 +396,21 @@ mod tests {
 
         // Test old accounts are still there
         (0..primordial_accounts.len()).for_each(|i| {
-            assert_eq!(genesis_block.accounts[i].0, primordial_accounts[i].pubkey);
             assert_eq!(
+                primordial_accounts[&genesis_block.accounts[i].0.to_string()],
                 genesis_block.accounts[i].1.lamports,
-                primordial_accounts[i].lamports
             );
         });
 
         // Test new account data matches
         (0..primordial_accounts1.len()).for_each(|i| {
             assert_eq!(
-                genesis_block.accounts[primordial_accounts.len() + i].0,
+                primordial_accounts1[&genesis_block.accounts[primordial_accounts.len() + i]
-                primordial_accounts1[i].pubkey
+                    .0
-            );
+                    .to_string()],
-            assert_eq!(
                 genesis_block.accounts[primordial_accounts.len() + i]
                     .1
                     .lamports,
-                primordial_accounts1[i].lamports
             );
         });
     }

multinode-demo/fullnode.sh

@@ -29,6 +29,7 @@ Start a validator or a replicator
   --no-voting               - start node without vote signer
   --rpc-port port           - custom RPC port for this node
   --no-restart              - do not restart the node if it exits
+  --no-airdrop              - The genesis block has an account for the node. Airdrops are not required.
 
 EOF
   exit 1
@@ -97,9 +98,14 @@ setup_validator_accounts() {
   if [[ -f $configured_flag ]]; then
     echo "Vote and stake accounts have already been configured"
   else
+    if ((airdrops_enabled)); then
       # Fund the node with enough tokens to fund its Vote, Staking, and Storage accounts
       declare fees=100 # TODO: No hardcoded transaction fees, fetch the current cluster fees
       $solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop $((node_lamports+stake_lamports+fees)) || return $?
+    else
+      echo "current account balance is "
+      $solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $?
+    fi
 
     # Fund the vote account from the node, with the node as the node_pubkey
     $solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \
@@ -149,7 +155,12 @@ setup_replicator_account() {
   if [[ -f $configured_flag ]]; then
     echo "Replicator account has already been configured"
   else
+    if ((airdrops_enabled)); then
       $solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" airdrop "$node_lamports" || return $?
+    else
+      echo "current account balance is "
+      $solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" balance || return $?
+    fi
 
     # Setup replicator storage account
     $solana_wallet --keypair "$node_keypair_path" --url "http://$entrypoint_ip:8899" \
@@ -179,6 +190,7 @@ poll_for_new_genesis_block=0
 label=
 identity_keypair_path=
 no_restart=0
+airdrops_enabled=1
 
 positional_args=()
 while [[ -n $1 ]]; do
@@ -233,6 +245,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --gossip-port ]]; then
       args+=("$1" "$2")
       shift 2
+    elif [[ $1 = --no-airdrop ]]; then
+      airdrops_enabled=0
+      shift
     elif [[ $1 = -h ]]; then
       fullnode_usage "$@"
     else

net/net.sh

@@ -55,6 +55,12 @@ Operate a configured testnet
                                       - Override the default --hashes-per-tick for the cluster
    -n NUM_FULL_NODES                  - Number of fullnodes to apply command to.
 
+   -x Accounts and Stakes for external nodes
+                                      - A YML file with a list of account pubkeys and corresponding stakes
+                                         for external nodes
+   -s Num lamports per node in genesis block
+                                      - Create account keypairs for internal nodes and assign these many lamports
+
  sanity/start/update-specific options:
    -F                   - Discard validator nodes that didn't bootup successfully
    -o noLedgerVerify    - Skip ledger verification
@@ -89,6 +95,8 @@ benchExchangeExtraArgs=
 failOnValidatorBootupFailure=true
 genesisOptions=
 numFullnodesRequested=
+externalPrimordialAccountsFile=
+stakeNodesInGenesisBlock=
 
 command=$1
 [[ -n $command ]] || usage
@@ -112,7 +120,7 @@ while [[ -n $1 ]]; do
   fi
 done
 
-while getopts "h?T:t:o:f:rD:c:Fn:" opt "${shortArgs[@]}"; do
+while getopts "h?T:t:o:f:rD:c:Fn:x:s:" opt "${shortArgs[@]}"; do
   case $opt in
   h | \?)
     usage
@@ -191,6 +199,12 @@ while getopts "h?T:t:o:f:rD:c:Fn:" opt "${shortArgs[@]}"; do
   F)
     failOnValidatorBootupFailure=false
     ;;
+  x)
+    externalPrimordialAccountsFile=$OPTARG
+    ;;
+  s)
+    stakeNodesInGenesisBlock=$OPTARG
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -291,6 +305,7 @@ startCommon() {
 startBootstrapLeader() {
   declare ipAddress=$1
   declare logFile="$2"
+  declare nodeIndex="$3"
   echo "--- Starting bootstrap leader: $ipAddress"
   echo "start log: $logFile"
 
@@ -299,6 +314,8 @@ startBootstrapLeader() {
   (
     set -x
     startCommon "$ipAddress" || exit 1
+    [[ -z "$externalPrimordialAccountsFile" ]] || rsync -vPrc -e "ssh ${sshOptions[*]}" "$externalPrimordialAccountsFile" \
+      "$ipAddress:~/solana/config/external-primodial-accounts.yml"
     case $deployMethod in
     tar)
       rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/bin/* "$ipAddress:~/.cargo/bin/"
@@ -316,10 +333,13 @@ startBootstrapLeader() {
          $deployMethod \
          bootstrap-leader \
          $entrypointIp \
-         $((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]})) \
+         $((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]} + ${#replicatorIpList[@]})) \
          \"$RUST_LOG\" \
          $skipSetup \
          $failOnValidatorBootupFailure \
+         \"$externalPrimordialAccountsFile\" \
+         \"$stakeNodesInGenesisBlock\" \
+         $nodeIndex \
          \"$genesisOptions\" \
       "
   ) >> "$logFile" 2>&1 || {
@@ -332,6 +352,7 @@ startBootstrapLeader() {
 startNode() {
   declare ipAddress=$1
   declare nodeType=$2
+  declare nodeIndex="$3"
   declare logFile="$netLogDir/fullnode-$ipAddress.log"
 
   echo "--- Starting $nodeType: $ipAddress"
@@ -344,10 +365,13 @@ startNode() {
          $deployMethod \
          $nodeType \
          $entrypointIp \
-         $((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]})) \
+         $((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]} + ${#replicatorIpList[@]})) \
          \"$RUST_LOG\" \
          $skipSetup \
          $failOnValidatorBootupFailure \
+         \"$externalPrimordialAccountsFile\" \
+         \"$stakeNodesInGenesisBlock\" \
+         $nodeIndex \
          \"$genesisOptions\" \
       "
   ) >> "$logFile" 2>&1 &
@@ -492,7 +516,7 @@ start() {
     if $bootstrapLeader; then
       SECONDS=0
       declare bootstrapNodeDeployTime=
-      startBootstrapLeader "$ipAddress" "$netLogDir/bootstrap-leader-$ipAddress.log"
+      startBootstrapLeader "$ipAddress" "$netLogDir/bootstrap-leader-$ipAddress.log" $loopCount
       bootstrapNodeDeployTime=$SECONDS
       $metricsWriteDatapoint "testnet-deploy net-bootnode-leader-started=1"
 
@@ -500,7 +524,7 @@ start() {
       SECONDS=0
       pids=()
     else
-      startNode "$ipAddress" $nodeType
+      startNode "$ipAddress" $nodeType $loopCount
 
       # Stagger additional node start time. If too many nodes start simultaneously
       # the bootstrap node gets more rsync requests from the additional nodes than

net/remote/remote-node.sh

@@ -11,7 +11,10 @@ numNodes="$4"
 RUST_LOG="$5"
 skipSetup="$6"
 failOnValidatorBootupFailure="$7"
-genesisOptions="$8"
+externalPrimordialAccountsFile="$8"
+stakeNodesInGenesisBlock="$9"
+nodeIndex="${10}"
+genesisOptions="${11}"
 set +x
 export RUST_LOG
 
@@ -77,6 +80,20 @@ local|tar)
       export SOLANA_CUDA=1
     fi
     set -x
+    rm -rf ./solana-node-keys
+    rm -rf ./solana-node-stakes
+    mkdir ./solana-node-stakes
+    if [[ -n $stakeNodesInGenesisBlock ]]; then
+      for i in $(seq 0 "$numNodes"); do
+        solana-keygen new -o ./solana-node-keys/"$i"
+        pubkey="$(solana-keygen pubkey ./solana-node-keys/"$i")"
+        echo "${pubkey}: $stakeNodesInGenesisBlock" >> ./solana-node-stakes/fullnode-stakes.yml
+      done
+    fi
+    [[ -z $externalPrimordialAccountsFile ]] || cat "$externalPrimordialAccountsFile" >> ./solana-node-stakes/fullnode-stakes.yml
+    if [ -f ./solana-node-stakes/fullnode-stakes.yml ]; then
+      genesisOptions+=" --primordial-accounts-file ./solana-node-stakes/fullnode-stakes.yml"
+    fi
     if [[ $skipSetup != true ]]; then
       args=(
         --bootstrap-leader-stake-lamports "$stake"
@@ -92,11 +109,17 @@ local|tar)
       --gossip-port "$entrypointIp":8001
     )
 
+    if [[ -n $stakeNodesInGenesisBlock ]]; then
+      args+=(--no-airdrop)
+    fi
     nohup ./multinode-demo/validator.sh --bootstrap-leader "${args[@]}" > fullnode.log 2>&1 &
     sleep 1
     ;;
   validator|blockstreamer)
     net/scripts/rsync-retry.sh -vPrc "$entrypointIp":~/.cargo/bin/ ~/.cargo/bin/
+    rm -f ~/solana/fullnode-identity.json
+    [[ -z $stakeNodesInGenesisBlock ]] || net/scripts/rsync-retry.sh -vPrc \
+    "$entrypointIp":~/solana/solana-node-keys/"$nodeIndex" ~/solana/fullnode-identity.json
 
     if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-validator-cuda ]]; then
       echo Selecting solana-validator-cuda
@@ -119,6 +142,14 @@ local|tar)
       args+=(--enable-rpc-exit)
     fi
 
+    if [[ -f ~/solana/fullnode-identity.json ]]; then
+      args+=(--identity ~/solana/fullnode-identity.json)
+    fi
+
+    if [[ -n $stakeNodesInGenesisBlock ]]; then
+      args+=(--no-airdrop)
+    fi
+
     set -x
     if [[ $skipSetup != true ]]; then
       ./multinode-demo/clear-config.sh
@@ -157,6 +188,11 @@ local|tar)
     args=(
       "$entrypointIp":~/solana "$entrypointIp:8001"
     )
+
+    if [[ -n $stakeNodesInGenesisBlock ]]; then
+      args+=(--no-airdrop)
+    fi
+
     if [[ $skipSetup != true ]]; then
       ./multinode-demo/clear-config.sh
     fi