Download and install SPL programs in run.sh/multinode-demo/docker (bp #11271) (#11275)

* Add --bpf-program argument

(cherry picked from commit 8eb6cbf784)

* Fetch and install SPL programs by default

(cherry picked from commit 3a1ca4efff)

* .sh

Co-authored-by: Michael Vines <mvines@gmail.com>

.gitignore

@@ -23,3 +23,7 @@ log-*/
 /.idea/
 /solana.iml
 /.vscode/
+
+# fetch-spl.sh artifacts
+/spl-genesis-args.sh
+/spl_*.so

fetch-spl.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Fetches the latest SPL programs and produces the solana-genesis command-line
+# arguments needed to install them
+#
+
+set -e
+
+fetch_program() {
+  declare name=$1
+  declare version=$2
+  declare address=$3
+
+  declare so=spl_$name-$version.so
+
+  genesis_args+=(--bpf-program "$address" "$so")
+
+  if [[ -r $so ]]; then
+    return
+  fi
+
+  if [[ -r ~/.cache/solana-spl/$so ]]; then
+    cp ~/.cache/solana-spl/"$so" "$so"
+  else
+    echo "Downloading $name $version"
+    (
+      set -x
+      curl -L --retry 5 --retry-delay 2 --retry-connrefused \
+        -o "$so" \
+        "https://github.com/solana-labs/solana-program-library/releases/download/$name-v$version/spl_$name.so"
+    )
+
+    mkdir -p ~/.cache/solana-spl
+    cp "$so" ~/.cache/solana-spl/"$so"
+  fi
+
+}
+
+fetch_program token 1.0.0 TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o
+fetch_program memo  1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo
+
+echo "${genesis_args[@]}" > spl-genesis-args.sh
+
+echo
+echo "Available SPL programs:"
+ls -l spl_*.so
+
+echo
+echo "solana-genesis command-line arguments (spl-genesis-args.sh):"
+cat spl-genesis-args.sh

genesis/src/main.rs

@@ -12,7 +12,7 @@ use solana_ledger::{
 };
 use solana_sdk::{
     account::Account,
-    clock,
+    bpf_loader, clock,
     epoch_schedule::EpochSchedule,
     fee_calculator::FeeRateGovernor,
     genesis_config::{GenesisConfig, OperatingMode},
@@ -26,7 +26,14 @@ use solana_sdk::{
 use solana_stake_program::stake_state::{self, StakeState};
 use solana_vote_program::vote_state::{self, VoteState};
 use std::{
-    collections::HashMap, error, fs::File, io, path::PathBuf, process, str::FromStr, time::Duration,
+    collections::HashMap,
+    error,
+    fs::File,
+    io::{self, Read},
+    path::PathBuf,
+    process,
+    str::FromStr,
+    time::Duration,
 };
 
 pub enum AccountFileFormat {
@@ -341,6 +348,15 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                     "maximum total uncompressed file size of created genesis archive",
                 ),
         )
+        .arg(
+            Arg::with_name("bpf_program")
+                .long("bpf-program")
+                .value_name("ADDRESS BPF_PROGRAM.SO")
+                .takes_value(true)
+                .number_of_values(2)
+                .multiple(true)
+                .help("Install a BPF program at the given address"),
+        )
         .get_matches();
 
     let faucet_lamports = value_t!(matches, "faucet_lamports", u64).unwrap_or(0);
@@ -535,6 +551,39 @@ fn main() -> Result<(), Box<dyn error::Error>> {
 
     add_genesis_accounts(&mut genesis_config, issued_lamports - faucet_lamports);
 
+    if let Some(values) = matches.values_of("bpf_program") {
+        let values: Vec<&str> = values.collect::<Vec<_>>();
+        for address_program in values.chunks(2) {
+            match address_program {
+                [address, program] => {
+                    let address = address.parse::<Pubkey>().unwrap_or_else(|err| {
+                        eprintln!("Error: invalid address {}: {}", address, err);
+                        process::exit(1);
+                    });
+
+                    let mut program_data = vec![];
+                    File::open(program)
+                        .and_then(|mut file| file.read_to_end(&mut program_data))
+                        .unwrap_or_else(|err| {
+                            eprintln!("Error: failed to read {}: {}", program, err);
+                            process::exit(1);
+                        });
+                    genesis_config.add_account(
+                        address,
+                        Account {
+                            lamports: genesis_config.rent.minimum_balance(program_data.len()),
+                            data: program_data,
+                            executable: true,
+                            owner: bpf_loader::id(),
+                            rent_epoch: 0,
+                        },
+                    );
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
     solana_logger::setup();
     create_new_ledger(
         &ledger_path,

multinode-demo/setup.sh

@@ -33,9 +33,19 @@ args=(
                         "$SOLANA_CONFIG_DIR"/bootstrap-validator/vote-account.json
                         "$SOLANA_CONFIG_DIR"/bootstrap-validator/stake-account.json
 )
+
+"$SOLANA_ROOT"/fetch-spl.sh
+if [[ -r spl-genesis-args.sh ]]; then
+  SPL_GENESIS_ARGS=$(cat "$SOLANA_ROOT"/spl-genesis-args.sh)
+  #shellcheck disable=SC2207
+  #shellcheck disable=SC2206
+  args+=($SPL_GENESIS_ARGS)
+fi
+
 default_arg --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator
 default_arg --faucet-pubkey "$SOLANA_CONFIG_DIR"/faucet.json
 default_arg --faucet-lamports 500000000000000000
 default_arg --hashes-per-tick auto
 default_arg --operating-mode development
+
 $solana_genesis "${args[@]}"

net/net.sh

@@ -222,7 +222,7 @@ syncScripts() {
   declare ipAddress=$1
   rsync -vPrc -e "ssh ${sshOptions[*]}" \
     --exclude 'net/log*' \
-    "$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
+    "$SOLANA_ROOT"/{fetch-perf-libs.sh,fetch-spl.sh,scripts,net,multinode-demo} \
     "$ipAddress":~/solana/ > /dev/null
 }
 

run.sh

@@ -63,6 +63,11 @@ fi
 if [[ -e "$ledgerDir"/genesis.bin || -e "$ledgerDir"/genesis.tar.bz2 ]]; then
   echo "Use existing genesis"
 else
+  ./fetch-spl.sh
+  if [[ -r spl-genesis-args.sh ]]; then
+    SPL_GENESIS_ARGS=$(cat spl-genesis-args.sh)
+  fi
+
   # shellcheck disable=SC2086
   solana-genesis \
     --hashes-per-tick sleep \
@@ -74,6 +79,7 @@ else
       "$dataDir"/validator-stake-account.json \
     --ledger "$ledgerDir" \
     --operating-mode development \
+    $SPL_GENESIS_ARGS \
     $SOLANA_RUN_SH_GENESIS_ARGS
 fi
 

sdk/docker-solana/build.sh

@@ -24,6 +24,11 @@ rm -rf usr/
   scripts/cargo-install-all.sh --use-move sdk/docker-solana/usr
 
 cp -f ../../run.sh usr/bin/solana-run.sh
+cp -f ../../fetch-spl.sh usr/bin/
+(
+  cd usr/bin
+  ./fetch-spl.sh
+)
 
 docker build -t solanalabs/solana:"$CHANNEL_OR_TAG" .