gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixes for storage program and rework storage stage (#4654)

Browse Source 
automerge
This commit is contained in:
Sagar Dhawan
2019-06-11 18:27:47 -07:00
committed by Grimes
parent 575a897ffc
commit 8019bff391
11 changed files with 394 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
programs/storage_api/src/lib.rs
View File

@@ -5,7 +5,7 @@ pub mod storage_processor;
pub const SLOTS_PER_SEGMENT: u64 = 16;
pub fn get_segment_from_slot(slot: u64) -> usize {
(slot / SLOTS_PER_SEGMENT) as usize
((slot + (SLOTS_PER_SEGMENT - 1)) / SLOTS_PER_SEGMENT) as usize
}
const STORAGE_PROGRAM_ID: [u8; 32] = [

69
programs/storage_api/src/storage_contract.rs
View File

@@ -32,6 +32,8 @@ impl Default for ProofStatus {
pub struct Proof {
pub signature: Signature,
pub sha_state: Hash,
/// The start index of the segment proof is for
pub segment_index: usize,
}
#[derive(Default, Debug, Serialize, Deserialize, Clone)]
@@ -134,13 +136,12 @@ impl<'a> StorageAccount<'a> {
pub fn submit_mining_proof(
&mut self,
sha_state: Hash,
slot: u64,
segment_index: usize,
signature: Signature,
current_slot: u64,
) -> Result<(), InstructionError> {
let mut storage_contract = &mut self.account.state()?;
if let StorageContract::ReplicatorStorage { proofs, .. } = &mut storage_contract {
let segment_index = get_segment_from_slot(slot);
let current_segment = get_segment_from_slot(current_slot);
if segment_index >= current_segment {
@@ -149,11 +150,12 @@ impl<'a> StorageAccount<'a> {
}
debug!(
"Mining proof submitted with contract {:?} slot: {}",
sha_state, slot
"Mining proof submitted with contract {:?} segment_index: {}",
sha_state, segment_index
);
let segment_proofs = proofs.entry(segment_index).or_default();
// store the proofs in the "current" segment's entry in the hash map.
let segment_proofs = proofs.entry(current_segment).or_default();
if segment_proofs.contains_key(&sha_state) {
// do not accept duplicate proofs
return Err(InstructionError::InvalidArgument);
@@ -163,8 +165,11 @@ impl<'a> StorageAccount<'a> {
Proof {
sha_state,
signature,
segment_index,
},
);
// TODO check for time correctness
proofs.retain(|segment, _| *segment >= current_segment.saturating_sub(5));
self.account.set_state(storage_contract)
} else {
@@ -286,33 +291,20 @@ impl<'a> StorageAccount<'a> {
pub fn claim_storage_reward(
&mut self,
mining_pool: &mut KeyedAccount,
slot: u64,
current_slot: u64,
) -> Result<(), InstructionError> {
let mut storage_contract = &mut self.account.state()?;
if let StorageContract::ValidatorStorage {
reward_validations,
slot: state_slot,
..
reward_validations, ..
} = &mut storage_contract
{
let state_segment = get_segment_from_slot(*state_slot);
let claim_segment = get_segment_from_slot(slot);
if state_segment <= claim_segment || !reward_validations.contains_key(&claim_segment) {
debug!(
"current {:?}, claim {:?}, have rewards for {:?} segments",
state_segment,
claim_segment,
reward_validations.len()
);
return Err(InstructionError::InvalidArgument);
}
let num_validations = count_valid_proofs(
&reward_validations
.remove(&claim_segment)
.map(|mut proofs| proofs.drain().map(|(_, proof)| proof).collect::<Vec<_>>())
.unwrap_or_default(),
.drain()
.flat_map(|(_segment, mut proofs)| {
proofs.drain().map(|(_, proof)| proof).collect::<Vec<_>>()
})
.collect::<Vec<_>>(),
);
let reward = TOTAL_VALIDATOR_REWARDS * num_validations;
mining_pool.account.lamports -= reward;
@@ -324,34 +316,20 @@ impl<'a> StorageAccount<'a> {
..
} = &mut storage_contract
{
// if current tick height is a full segment away, allow reward collection
let claim_index = get_segment_from_slot(current_slot);
let claim_segment = get_segment_from_slot(slot);
// Todo this might might always be true
if claim_index <= claim_segment
|| !reward_validations.contains_key(&claim_segment)
|| !proofs.contains_key(&claim_segment)
{
info!(
"current {:?}, claim {:?}, have rewards for {:?} segments",
claim_index,
claim_segment,
reward_validations.len()
);
return Err(InstructionError::InvalidArgument);
}
// remove proofs for which rewards have already been collected
let segment_proofs = proofs.get_mut(&claim_segment).unwrap();
let segment_proofs = proofs;
let checked_proofs = reward_validations
.remove(&claim_segment)
.map(|mut proofs| {
.drain()
.flat_map(|(segment, mut proofs)| {
proofs
.drain()
.map(|(sha_state, proof)| {
proof
.into_iter()
.map(|proof| {
segment_proofs.remove(&sha_state);
segment_proofs.get_mut(&segment).and_then(|segment_proofs| {
segment_proofs.remove(&sha_state)
});
proof
})
.collect::<Vec<_>>()
@@ -359,7 +337,7 @@ impl<'a> StorageAccount<'a> {
.flatten()
.collect::<Vec<_>>()
})
.unwrap_or_default();
.collect::<Vec<_>>();
let total_proofs = checked_proofs.len() as u64;
let num_validations = count_valid_proofs(&checked_proofs);
let reward =
@@ -491,6 +469,7 @@ mod tests {
let proof = Proof {
signature: Signature::default(),
sha_state: Hash::default(),
segment_index,
};
let mut checked_proof = CheckedProof {
proof: proof.clone(),

25
programs/storage_api/src/storage_instruction.rs
View File

@@ -5,7 +5,7 @@ use solana_sdk::hash::Hash;
use solana_sdk::instruction::{AccountMeta, Instruction};
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::Signature;
use solana_sdk::syscall::tick_height;
use solana_sdk::syscall::current;
use solana_sdk::system_instruction;
use std::collections::HashMap;
@@ -25,7 +25,7 @@ pub enum StorageInstruction {
SubmitMiningProof {
sha_state: Hash,
slot: u64,
segment_index: usize,
signature: Signature,
},
AdvertiseStorageRecentBlockhash {
@@ -37,9 +37,7 @@ pub enum StorageInstruction {
/// Expects 1 Account:
/// 0 - Storage account with credits to redeem
/// 1 - MiningPool account to redeem credits from
ClaimStorageReward {
slot: u64,
},
ClaimStorageReward,
ProofValidation {
segment: u64,
proofs: Vec<(Pubkey, Vec<CheckedProof>)>,
@@ -118,17 +116,17 @@ pub fn create_mining_pool_account(
pub fn mining_proof(
storage_pubkey: &Pubkey,
sha_state: Hash,
slot: u64,
segment_index: usize,
signature: Signature,
) -> Instruction {
let storage_instruction = StorageInstruction::SubmitMiningProof {
sha_state,
slot,
segment_index,
signature,
};
let account_metas = vec![
AccountMeta::new(*storage_pubkey, true),
AccountMeta::new(tick_height::id(), false),
AccountMeta::new(current::id(), false),
];
Instruction::new(id(), &storage_instruction, account_metas)
}
@@ -144,7 +142,7 @@ pub fn advertise_recent_blockhash(
};
let account_metas = vec![
AccountMeta::new(*storage_pubkey, true),
AccountMeta::new(tick_height::id(), false),
AccountMeta::new(current::id(), false),
];
Instruction::new(id(), &storage_instruction, account_metas)
}
@@ -164,16 +162,11 @@ pub fn proof_validation<S: std::hash::BuildHasher>(
Instruction::new(id(), &storage_instruction, account_metas)
}
pub fn claim_reward(
storage_pubkey: &Pubkey,
mining_pool_pubkey: &Pubkey,
slot: u64,
) -> Instruction {
let storage_instruction = StorageInstruction::ClaimStorageReward { slot };
pub fn claim_reward(storage_pubkey: &Pubkey, mining_pool_pubkey: &Pubkey) -> Instruction {
let storage_instruction = StorageInstruction::ClaimStorageReward;
let account_metas = vec![
AccountMeta::new(*storage_pubkey, false),
AccountMeta::new(*mining_pool_pubkey, false),
AccountMeta::new(tick_height::id(), false),
];
Instruction::new(id(), &storage_instruction, account_metas)
}

33
programs/storage_api/src/storage_processor.rs
View File

@@ -6,8 +6,7 @@ use crate::storage_instruction::StorageInstruction;
use solana_sdk::account::KeyedAccount;
use solana_sdk::instruction::InstructionError;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::syscall::tick_height::TickHeight;
use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
use solana_sdk::syscall::current::Current;
pub fn process_instruction(
_program_id: &Pubkey,
@@ -41,43 +40,29 @@ pub fn process_instruction(
}
StorageInstruction::SubmitMiningProof {
sha_state,
slot,
segment_index,
signature,
} => {
if me_unsigned || rest.len() != 1 {
// This instruction must be signed by `me`
Err(InstructionError::InvalidArgument)?;
}
let tick_height = TickHeight::from(&rest[0].account).unwrap();
storage_account.submit_mining_proof(
sha_state,
slot,
signature,
tick_height / DEFAULT_TICKS_PER_SLOT,
)
let current = Current::from(&rest[0].account).unwrap();
storage_account.submit_mining_proof(sha_state, segment_index, signature, current.slot)
}
StorageInstruction::AdvertiseStorageRecentBlockhash { hash, slot } => {
if me_unsigned || rest.len() != 1 {
// This instruction must be signed by `me`
Err(InstructionError::InvalidArgument)?;
}
let tick_height = TickHeight::from(&rest[0].account).unwrap();
storage_account.advertise_storage_recent_blockhash(
hash,
slot,
tick_height / DEFAULT_TICKS_PER_SLOT,
)
let current = Current::from(&rest[0].account).unwrap();
storage_account.advertise_storage_recent_blockhash(hash, slot, current.slot)
}
StorageInstruction::ClaimStorageReward { slot } => {
if rest.len() != 2 {
StorageInstruction::ClaimStorageReward => {
if rest.len() != 1 {
Err(InstructionError::InvalidArgument)?;
}
let tick_height = TickHeight::from(&rest[1].account).unwrap();
storage_account.claim_storage_reward(
&mut rest[0],
slot,
tick_height / DEFAULT_TICKS_PER_SLOT,
)
storage_account.claim_storage_reward(&mut rest[0])
}
StorageInstruction::ProofValidation { segment, proofs } => {
if me_unsigned || rest.is_empty() {

141
programs/storage_program/tests/storage_processor.rs
View File

@@ -12,8 +12,8 @@ use solana_sdk::instruction::{Instruction, InstructionError};
use solana_sdk::message::Message;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
use solana_sdk::syscall::tick_height;
use solana_sdk::syscall::tick_height::TickHeight;
use solana_sdk::syscall::current;
use solana_sdk::syscall::current::Current;
use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT;
use solana_storage_api::storage_contract::StorageAccount;
use solana_storage_api::storage_contract::{
@@ -114,18 +114,22 @@ fn test_proof_bounds() {
.unwrap();
}
let ix = storage_instruction::mining_proof(
&pubkey,
Hash::default(),
SLOTS_PER_SEGMENT,
Signature::default(),
let ix = storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default());
// the proof is for segment 0, need to move the slot into segment 2
let mut current_account = current::create_account(1);
Current::to(
&Current {
slot: SLOTS_PER_SEGMENT * 2,
epoch: 0,
stakers_epoch: 0,
},
&mut current_account,
);
// the proof is for slot 16, which is in segment 0, need to move the tick height into segment 2
let ticks_till_next_segment = TICKS_IN_SEGMENT * 2;
let mut tick_account = tick_height::create_account(1);
TickHeight::to(ticks_till_next_segment, &mut tick_account);
assert_eq!(test_instruction(&ix, &mut [account, tick_account]), Ok(()));
assert_eq!(
test_instruction(&ix, &mut [account, current_account]),
Ok(())
);
}
#[test]
@@ -142,9 +146,9 @@ fn test_serialize_overflow() {
let tick_pubkey = Pubkey::new_rand();
let mut keyed_accounts = Vec::new();
let mut user_account = Account::default();
let mut tick_account = tick_height::create_account(1);
let mut current_account = current::create_account(1);
keyed_accounts.push(KeyedAccount::new(&pubkey, true, &mut user_account));
keyed_accounts.push(KeyedAccount::new(&tick_pubkey, false, &mut tick_account));
keyed_accounts.push(KeyedAccount::new(&tick_pubkey, false, &mut current_account));
let ix = storage_instruction::advertise_recent_blockhash(
&pubkey,
@@ -165,13 +169,19 @@ fn test_invalid_accounts_len() {
let ix = storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default());
// move tick height into segment 1
let ticks_till_next_segment = TICKS_IN_SEGMENT + 1;
let mut tick_account = tick_height::create_account(1);
TickHeight::to(ticks_till_next_segment, &mut tick_account);
let mut current_account = current::create_account(1);
Current::to(
&Current {
slot: 16,
epoch: 0,
stakers_epoch: 0,
},
&mut current_account,
);
assert!(test_instruction(&ix, &mut accounts).is_err());
let mut accounts = [Account::default(), tick_account, Account::default()];
let mut accounts = [Account::default(), current_account, Account::default()];
assert!(test_instruction(&ix, &mut accounts).is_err());
}
@@ -205,18 +215,30 @@ fn test_submit_mining_ok() {
}
let ix = storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default());
// move tick height into segment 1
let ticks_till_next_segment = TICKS_IN_SEGMENT + 1;
let mut tick_account = tick_height::create_account(1);
TickHeight::to(ticks_till_next_segment, &mut tick_account);
// move slot into segment 1
let mut current_account = current::create_account(1);
Current::to(
&Current {
slot: SLOTS_PER_SEGMENT,
epoch: 0,
stakers_epoch: 0,
},
&mut current_account,
);
assert_matches!(test_instruction(&ix, &mut [account, tick_account]), Ok(_));
assert_matches!(
test_instruction(&ix, &mut [account, current_account]),
Ok(_)
);
}
#[test]
fn test_validate_mining() {
solana_logger::setup();
let (genesis_block, mint_keypair) = create_genesis_block(1000);
let (mut genesis_block, mint_keypair) = create_genesis_block(1000);
genesis_block
.native_instruction_processors
.push(solana_storage_program::solana_storage_program!());
let mint_pubkey = mint_keypair.pubkey();
let replicator_1_storage_keypair = Keypair::new();
@@ -231,10 +253,8 @@ fn test_validate_mining() {
let mining_pool_keypair = Keypair::new();
let mining_pool_pubkey = mining_pool_keypair.pubkey();
let mut bank = Bank::new(&genesis_block);
bank.add_instruction_processor(id(), process_instruction);
let bank = Bank::new(&genesis_block);
let bank = Arc::new(bank);
let slot = 0;
let bank_client = BankClient::new_shared(&bank);
init_storage_accounts(
@@ -251,11 +271,13 @@ fn test_validate_mining() {
));
bank_client.send_message(&[&mint_keypair], message).unwrap();
// tick the bank up until it's moved into storage segment 2 because the next advertise is for segment 1
let next_storage_segment_tick_height = TICKS_IN_SEGMENT * 2;
for _ in 0..next_storage_segment_tick_height {
bank.register_tick(&bank.last_blockhash());
}
// create a new bank in segment 2
let bank = Arc::new(Bank::new_from_parent(
&bank,
&Pubkey::default(),
SLOTS_PER_SEGMENT * 2,
));
let bank_client = BankClient::new_shared(&bank);
// advertise for storage segment 1
let message = Message::new_with_payer(
@@ -273,15 +295,15 @@ fn test_validate_mining() {
// submit proofs 5 proofs for each replicator for segment 0
let mut checked_proofs: HashMap<_, Vec<_>> = HashMap::new();
for slot in 0..5 {
for _ in 0..5 {
checked_proofs
.entry(replicator_1_storage_id)
.or_default()
.push(submit_proof(
&mint_keypair,
&replicator_1_storage_keypair,
slot,
&bank_client,
0,
));
checked_proofs
.entry(replicator_2_storage_id)
@@ -289,8 +311,8 @@ fn test_validate_mining() {
.push(submit_proof(
&mint_keypair,
&replicator_2_storage_keypair,
slot,
&bank_client,
0,
));
}
let message = Message::new_with_payer(
@@ -302,10 +324,14 @@ fn test_validate_mining() {
Some(&mint_pubkey),
);
let next_storage_segment_tick_height = TICKS_IN_SEGMENT;
for _ in 0..next_storage_segment_tick_height {
bank.register_tick(&bank.last_blockhash());
}
// move banks into the next segment
let proof_segment = get_segment_from_slot(bank.slot());
let bank = Arc::new(Bank::new_from_parent(
&bank,
&Pubkey::default(),
SLOTS_PER_SEGMENT + bank.slot(),
));
let bank_client = BankClient::new_shared(&bank);
assert_matches!(
bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message),
@@ -315,7 +341,7 @@ fn test_validate_mining() {
let message = Message::new_with_payer(
vec![storage_instruction::proof_validation(
&validator_storage_id,
get_segment_from_slot(slot) as u64,
proof_segment as u64,
checked_proofs,
)],
Some(&mint_pubkey),
@@ -335,10 +361,13 @@ fn test_validate_mining() {
Some(&mint_pubkey),
);
let next_storage_segment_tick_height = TICKS_IN_SEGMENT;
for _ in 0..next_storage_segment_tick_height {
bank.register_tick(&bank.last_blockhash());
}
// move banks into the next segment
let bank = Arc::new(Bank::new_from_parent(
&bank,
&Pubkey::default(),
SLOTS_PER_SEGMENT + bank.slot(),
));
let bank_client = BankClient::new_shared(&bank);
assert_matches!(
bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message),
@@ -351,7 +380,6 @@ fn test_validate_mining() {
vec![storage_instruction::claim_reward(
&validator_storage_id,
&mining_pool_pubkey,
slot,
)],
Some(&mint_pubkey),
);
@@ -375,7 +403,6 @@ fn test_validate_mining() {
vec![storage_instruction::claim_reward(
&replicator_1_storage_id,
&mining_pool_pubkey,
slot,
)],
Some(&mint_pubkey),
);
@@ -385,7 +412,6 @@ fn test_validate_mining() {
vec![storage_instruction::claim_reward(
&replicator_2_storage_id,
&mining_pool_pubkey,
slot,
)],
Some(&mint_pubkey),
);
@@ -453,15 +479,15 @@ fn get_storage_slot<C: SyncClient>(client: &C, account: &Pubkey) -> u64 {
fn submit_proof(
mint_keypair: &Keypair,
storage_keypair: &Keypair,
slot: u64,
bank_client: &BankClient,
segment_index: u64,
) -> CheckedProof {
let sha_state = Hash::new(Pubkey::new_rand().as_ref());
let message = Message::new_with_payer(
vec![storage_instruction::mining_proof(
&storage_keypair.pubkey(),
sha_state,
slot,
segment_index as usize,
Signature::default(),
)],
Some(&mint_keypair.pubkey()),
@@ -475,6 +501,7 @@ fn submit_proof(
proof: Proof {
signature: Signature::default(),
sha_state,
segment_index: segment_index as usize,
},
status: ProofStatus::Valid,
}
@@ -497,20 +524,20 @@ fn get_storage_blockhash<C: SyncClient>(client: &C, account: &Pubkey) -> Hash {
#[test]
fn test_bank_storage() {
let (genesis_block, mint_keypair) = create_genesis_block(1000);
let (mut genesis_block, mint_keypair) = create_genesis_block(1000);
genesis_block
.native_instruction_processors
.push(solana_storage_program::solana_storage_program!());
let mint_pubkey = mint_keypair.pubkey();
let replicator_keypair = Keypair::new();
let replicator_pubkey = replicator_keypair.pubkey();
let validator_keypair = Keypair::new();
let validator_pubkey = validator_keypair.pubkey();
let mut bank = Bank::new(&genesis_block);
bank.add_instruction_processor(id(), process_instruction);
let bank = Bank::new(&genesis_block);
// tick the bank up until it's moved into storage segment 2
let next_storage_segment_tick_height = TICKS_IN_SEGMENT * 2;
for _ in 0..next_storage_segment_tick_height {
bank.register_tick(&bank.last_blockhash());
}
// create a new bank in storage segment 2
let bank = Bank::new_from_parent(&Arc::new(bank), &Pubkey::new_rand(), SLOTS_PER_SEGMENT * 2);
let bank_client = BankClient::new(bank);
let x = 42;
@@ -541,7 +568,7 @@ fn test_bank_storage() {
vec![storage_instruction::advertise_recent_blockhash(
&validator_pubkey,
storage_blockhash,
SLOTS_PER_SEGMENT,
SLOTS_PER_SEGMENT as u64,
)],
Some(&mint_pubkey),
);