make runtime depend on bpf_loader (#4601)

* make runtime depend on bpf_loader

* remove vote redundancy, move bpf_loader to genesis, export program\! from bpf_loader crate

* move bpf_loader specification into genesis

* bpf tests to use genesis with bpf

* need to avoid depending on programs, except for macros

Cargo.lock

@@ -2477,6 +2477,7 @@ dependencies = [
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana 0.16.0",
+ "solana-bpfloader 0.16.0",
  "solana-budget-api 0.16.0",
  "solana-budget-program 0.16.0",
  "solana-config-api 0.16.0",
@@ -2647,6 +2648,7 @@ dependencies = [
  "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-bpfloader 0.16.0",
  "solana-logger 0.16.0",
  "solana-metrics 0.16.0",
  "solana-noop-program 0.16.0",

chacha-sys/cpu-crypt/chacha_cbc.c

@@ -70,4 +70,3 @@ void chacha20_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t in_len,
 {
     chacha20_cbc128_encrypt(in, out, in_len, key, ivec);
 }
-

core/src/lib.rs

@@ -73,6 +73,9 @@ pub mod window_service;
 #[macro_use]
 extern crate solana_budget_program;
 
+#[macro_use]
+extern crate solana_storage_program;
+
 #[cfg(test)]
 #[macro_use]
 extern crate hex_literal;

core/src/local_cluster.rs

@@ -20,8 +20,8 @@ use solana_sdk::timing::DEFAULT_SLOTS_PER_EPOCH;
 use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::transaction::Transaction;
 use solana_stake_api::stake_instruction;
+use solana_storage_api::storage_contract;
 use solana_storage_api::storage_instruction;
-use solana_storage_program::genesis_block_util::GenesisBlockUtil;
 use solana_vote_api::vote_instruction;
 use solana_vote_api::vote_state::VoteState;
 use std::collections::HashMap;
@@ -130,8 +130,6 @@ impl LocalCluster {
             &leader_pubkey,
             config.node_stakes[0],
         );
-        let storage_keypair = Keypair::new();
-        genesis_block.add_storage_program(&leader_pubkey, &storage_keypair.pubkey());
         genesis_block.ticks_per_slot = config.ticks_per_slot;
         genesis_block.slots_per_epoch = config.slots_per_epoch;
         genesis_block.stakers_slot_offset = config.stakers_slot_offset;
@@ -140,6 +138,15 @@ impl LocalCluster {
             .native_instruction_processors
             .extend_from_slice(&config.native_instruction_processors);
 
+        let storage_keypair = Keypair::new();
+        genesis_block.accounts.push((
+            storage_keypair.pubkey(),
+            storage_contract::create_validator_storage_account(leader_pubkey, 1),
+        ));
+        genesis_block
+            .native_instruction_processors
+            .push(solana_storage_program!());
+
         let (genesis_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_block);
         let leader_ledger_path = tmp_copy_blocktree!(&genesis_ledger_path);
         let leader_contact_info = leader_node.info.clone();

genesis/Cargo.toml

@@ -16,6 +16,7 @@ serde_derive = "1.0.92"
 serde_json = "1.0.39"
 serde_yaml = "0.8.9"
 solana = { path = "../core", version = "0.16.0" }
+solana-bpfloader = { path = "../programs/bpf_loader", version = "0.16.0" }
 solana-budget-api = { path = "../programs/budget_api", version = "0.16.0" }
 solana-budget-program = { path = "../programs/budget_program", version = "0.16.0" }
 solana-config-api = { path = "../programs/config_api", version = "0.16.0" }

genesis/src/main.rs

@@ -1,5 +1,7 @@
 //! A command-line executable for generating the chain's genesis block.
 #[macro_use]
+extern crate solana_bpf_loader;
+#[macro_use]
 extern crate solana_vote_program;
 #[macro_use]
 extern crate solana_stake_program;
@@ -11,6 +13,8 @@ extern crate solana_token_program;
 extern crate solana_config_program;
 #[macro_use]
 extern crate solana_exchange_program;
+#[macro_use]
+extern crate solana_storage_program;
 
 use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg};
 use solana::blocktree::create_new_ledger;
@@ -24,7 +28,7 @@ use solana_sdk::signature::{read_keypair, KeypairUtil};
 use solana_sdk::system_program;
 use solana_sdk::timing;
 use solana_stake_api::stake_state;
-use solana_storage_program::genesis_block_util::GenesisBlockUtil;
+use solana_storage_api::storage_contract;
 use solana_vote_api::vote_state;
 use std::collections::HashMap;
 use std::error;
@@ -266,14 +270,23 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                     bootstrap_leader_stake_lamports,
                 ),
             ),
+            (
+                bootstrap_storage_keypair.pubkey(),
+                storage_contract::create_validator_storage_account(
+                    bootstrap_leader_keypair.pubkey(),
+                    1,
+                ),
+            ),
         ],
         &[
+            solana_bpf_loader!(),
             solana_vote_program!(),
             solana_stake_program!(),
             solana_budget_program!(),
             solana_token_program!(),
             solana_config_program!(),
             solana_exchange_program!(),
+            solana_storage_program!(),
         ],
     );
 
@@ -281,11 +294,6 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         append_primordial_accounts(file, &mut genesis_block)?;
     }
 
-    genesis_block.add_storage_program(
-        &bootstrap_leader_keypair.pubkey(),
-        &bootstrap_storage_keypair.pubkey(),
-    );
-
     genesis_block.fee_calculator.target_lamports_per_signature =
         value_t_or_exit!(matches, "target_lamports_per_signature", u64);
     genesis_block.fee_calculator.target_signatures_per_slot =

programs/bpf/tests/programs.rs

@@ -2,8 +2,8 @@
 mod bpf {
     use solana_runtime::bank::Bank;
     use solana_runtime::bank_client::BankClient;
+    use solana_runtime::genesis_utils::{create_genesis_block, GenesisBlockInfo};
     use solana_runtime::loader_utils::load_program;
-    use solana_sdk::genesis_block::create_genesis_block;
     use solana_sdk::native_loader;
     use std::env;
     use std::fs::File;
@@ -41,7 +41,11 @@ mod bpf {
             let mut elf = Vec::new();
             file.read_to_end(&mut elf).unwrap();
 
-            let (genesis_block, mint_keypair) = create_genesis_block(50);
+            let GenesisBlockInfo {
+                genesis_block,
+                mint_keypair,
+                ..
+            } = create_genesis_block(50);
             let bank = Bank::new(&genesis_block);
             let bank_client = BankClient::new(bank);
 
@@ -72,7 +76,11 @@ mod bpf {
                 let mut elf = Vec::new();
                 file.read_to_end(&mut elf).unwrap();
 
-                let (genesis_block, mint_keypair) = create_genesis_block(50);
+                let GenesisBlockInfo {
+                    genesis_block,
+                    mint_keypair,
+                    ..
+                } = create_genesis_block(50);
                 let bank = Bank::new(&genesis_block);
                 let bank_client = BankClient::new(bank);
 
@@ -119,7 +127,11 @@ mod bpf {
                 let mut elf = Vec::new();
                 file.read_to_end(&mut elf).unwrap();
 
-                let (genesis_block, mint_keypair) = create_genesis_block(50);
+                let GenesisBlockInfo {
+                    genesis_block,
+                    mint_keypair,
+                    ..
+                } = create_genesis_block(50);
                 let bank = Bank::new(&genesis_block);
                 let bank_client = BankClient::new(bank);
 

programs/bpf_loader/src/lib.rs

@@ -3,6 +3,16 @@ pub mod allocator_bump;
 pub mod allocator_system;
 pub mod bpf_verifier;
 
+#[macro_export]
+macro_rules! solana_bpf_loader {
+    () => {
+        (
+            "solana_bpf_loader".to_string(),
+            solana_sdk::bpf_loader::id(),
+        )
+    };
+}
+
 use alloc::Alloc;
 use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use libc::c_char;

programs/storage_program/src/genesis_block_util.rs

@@ -1,23 +0,0 @@
-use crate::solana_storage_program;
-use solana_sdk::genesis_block::GenesisBlock;
-use solana_sdk::pubkey::Pubkey;
-use solana_storage_api::storage_contract;
-
-pub trait GenesisBlockUtil {
-    fn add_storage_program(&mut self, validator_pubkey: &Pubkey, validator_storage_pubkey: &Pubkey);
-}
-
-impl GenesisBlockUtil for GenesisBlock {
-    fn add_storage_program(
-        &mut self,
-        validator_pubkey: &Pubkey,
-        validator_storage_pubkey: &Pubkey,
-    ) {
-        self.accounts.push((
-            *validator_storage_pubkey,
-            storage_contract::create_validator_storage_account(*validator_pubkey, 1),
-        ));
-        self.native_instruction_processors
-            .push(solana_storage_program!());
-    }
-}

programs/storage_program/src/lib.rs

@@ -1,5 +1,3 @@
-pub mod genesis_block_util;
-
 #[macro_export]
 macro_rules! solana_storage_program {
     () => {

runtime/Cargo.toml

@@ -26,6 +26,7 @@ serde_derive = "1.0.92"
 serde_json = "1.0.38"
 solana-logger = { path = "../logger", version = "0.16.0" }
 solana-metrics = { path = "../metrics", version = "0.16.0" }
+solana-bpfloader = { path = "../programs/bpf_loader", version = "0.16.0" }
 solana-noop-program = { path = "../programs/noop_program", version = "0.16.0" }
 solana-sdk = { path = "../sdk", version = "0.16.0" }
 solana-stake-api = { path = "../programs/stake_api", version = "0.16.0" }

runtime/src/bank.rs

@@ -503,14 +503,6 @@ impl Bank {
             "solana_system_program",
             &solana_sdk::system_program::id(),
         );
-        self.register_native_instruction_processor(
-            "solana_bpf_loader",
-            &solana_sdk::bpf_loader::id(),
-        );
-        self.register_native_instruction_processor(
-            &solana_vote_program!().0,
-            &solana_vote_program!().1,
-        );
 
         // Add additional native programs specified in the genesis block
         for (name, program_id) in &genesis_block.native_instruction_processors {

runtime/src/genesis_utils.rs

@@ -15,6 +15,10 @@ pub struct GenesisBlockInfo {
     pub voting_keypair: Keypair,
 }
 
+pub fn create_genesis_block(mint_lamports: u64) -> GenesisBlockInfo {
+    create_genesis_block_with_leader(mint_lamports, &Pubkey::new_rand(), 0)
+}
+
 pub fn create_genesis_block_with_leader(
     mint_lamports: u64,
     bootstrap_leader_pubkey: &Pubkey,
@@ -60,7 +64,11 @@ pub fn create_genesis_block_with_leader(
                 ),
             ),
         ],
-        &[solana_vote_program!(), solana_stake_program!()],
+        &[
+            solana_bpf_loader!(),
+            solana_vote_program!(),
+            solana_stake_program!(),
+        ],
     );
 
     GenesisBlockInfo {

runtime/src/lib.rs

@@ -27,5 +27,8 @@ extern crate solana_vote_program;
 #[macro_use]
 extern crate solana_stake_program;
 
+#[macro_use]
+extern crate solana_bpf_loader;
+
 #[macro_use]
 extern crate serde_derive;