solana/programs/storage_api/src/storage_processor.rs

//! storage program
//!  Receive mining proofs from miners, validate the answers
//!  and give reward for good proofs.

use crate::storage_contract::{CheckedProof, Proof, ProofStatus, StorageContract};
use crate::storage_instruction::StorageInstruction;
use crate::{get_segment_from_entry, ENTRIES_PER_SEGMENT};
use log::*;
use serde::Serialize;
use solana_sdk::account::{Account, KeyedAccount};
use solana_sdk::hash::Hash;
use solana_sdk::instruction::InstructionError;
use solana_sdk::instruction::InstructionError::InvalidArgument;
use solana_sdk::pubkey::Pubkey;
use std::cmp;

pub const TOTAL_VALIDATOR_REWARDS: u64 = 1;
pub const TOTAL_REPLICATOR_REWARDS: u64 = 1;

fn count_valid_proofs(proofs: &[CheckedProof]) -> u64 {
    let mut num = 0;
    for proof in proofs {
        if let ProofStatus::Valid = proof.status {
            num += 1;
        }
    }
    num
}

/// Serialize account data
fn store_contract<T>(account: &mut Account, contract: &T) -> Result<(), InstructionError>
where
    T: Serialize,
{
    if bincode::serialize_into(&mut account.data[..], contract).is_err() {
        return Err(InstructionError::AccountDataTooSmall);
    }
    Ok(())
}

/// Deserialize account data
fn read_contract(account: &Account) -> Result<StorageContract, InstructionError> {
    if let Ok(storage_contract) = bincode::deserialize(&account.data) {
        Ok(storage_contract)
    } else {
        Err(InstructionError::InvalidAccountData)
    }
}

/// Deserialize account data but handle uninitialized accounts
fn read_contract_with_default(
    op: &StorageInstruction,
    account: &Account,
) -> Result<StorageContract, InstructionError> {
    let mut storage_contract = read_contract(&account);
    if let Ok(StorageContract::Default) = storage_contract {
        match op {
            StorageInstruction::SubmitMiningProof { .. } => {
                storage_contract = Ok(StorageContract::ReplicatorStorage {
                    proofs: vec![],
                    reward_validations: vec![],
                })
            }
            StorageInstruction::AdvertiseStorageRecentBlockhash { .. }
            | StorageInstruction::ProofValidation { .. } => {
                storage_contract = Ok(StorageContract::ValidatorStorage {
                    entry_height: 0,
                    hash: Hash::default(),
                    lockout_validations: vec![],
                    reward_validations: vec![],
                })
            }
            StorageInstruction::ClaimStorageReward { .. } => Err(InvalidArgument)?,
        }
    }
    storage_contract
}

/// Store the result of a proof validation into the replicator account
fn store_validation_result(
    account: &mut Account,
    segment_index: usize,
    status: ProofStatus,
) -> Result<(), InstructionError> {
    let mut storage_contract = read_contract(account)?;
    match &mut storage_contract {
        StorageContract::ReplicatorStorage {
            proofs,
            reward_validations,
            ..
        } => {
            if segment_index > reward_validations.len() || reward_validations.is_empty() {
                reward_validations.resize(cmp::max(1, segment_index), vec![]);
            }
            let result = proofs[segment_index].clone();
            reward_validations[segment_index].push(CheckedProof {
                proof: result,
                status,
            });
        }
        _ => return Err(InstructionError::InvalidAccountData),
    }
    store_contract(account, &storage_contract)?;
    Ok(())
}

pub fn process_instruction(
    _program_id: &Pubkey,
    keyed_accounts: &mut [KeyedAccount],
    data: &[u8],
    tick_height: u64,
) -> Result<(), InstructionError> {
    solana_logger::setup();

    // accounts_keys[0] must be signed
    if keyed_accounts[0].signer_key().is_none() {
        info!("account[0] is unsigned");
        Err(InstructionError::GenericError)?;
    }

    if let Ok(syscall) = bincode::deserialize(data) {
        let mut storage_contract =
            read_contract_with_default(&syscall, &keyed_accounts[0].account)?;
        match syscall {
            StorageInstruction::SubmitMiningProof {
                sha_state,
                entry_height,
                signature,
            } => {
                if let StorageContract::ReplicatorStorage { proofs, .. } = &mut storage_contract {
                    if keyed_accounts.len() != 1 {
                        // keyed_accounts[1] should be the main storage key
                        // to access its data
                        Err(InstructionError::InvalidArgument)?;
                    }

                    let segment_index = get_segment_from_entry(entry_height);
                    if segment_index > proofs.len() || proofs.is_empty() {
                        proofs.resize(cmp::max(1, segment_index), Proof::default());
                    }

                    if segment_index > proofs.len() {
                        // only possible if usize max < u64 max
                        return Err(InstructionError::InvalidArgument);
                    }

                    debug!(
                        "Mining proof submitted with contract {:?} entry_height: {}",
                        sha_state, entry_height
                    );

                    let proof_info = Proof {
                        id: *keyed_accounts[0].signer_key().unwrap(),
                        sha_state,
                        signature,
                    };
                    proofs[segment_index] = proof_info;
                } else {
                    Err(InstructionError::InvalidArgument)?;
                }
            }
            StorageInstruction::AdvertiseStorageRecentBlockhash { hash, entry_height } => {
                if let StorageContract::ValidatorStorage {
                    entry_height: state_entry_height,
                    hash: state_hash,
                    reward_validations,
                    lockout_validations,
                } = &mut storage_contract
                {
                    if keyed_accounts.len() != 1 {
                        // keyed_accounts[1] should be the main storage key
                        // to access its data
                        Err(InstructionError::InvalidArgument)?;
                    }

                    let original_segments = *state_entry_height / ENTRIES_PER_SEGMENT;
                    let segments = entry_height / ENTRIES_PER_SEGMENT;
                    debug!(
                        "advertise new last id segments: {} orig: {}",
                        segments, original_segments
                    );
                    if segments <= original_segments {
                        return Err(InstructionError::InvalidArgument);
                    }

                    *state_entry_height = entry_height;
                    *state_hash = hash;

                    // move lockout_validations to reward_validations
                    *reward_validations = lockout_validations.clone();
                    lockout_validations.clear();
                    lockout_validations.resize(segments as usize, Vec::new());
                } else {
                    return Err(InstructionError::InvalidArgument);
                }
            }
            StorageInstruction::ClaimStorageReward { entry_height } => {
                if keyed_accounts.len() != 1 {
                    // keyed_accounts[1] should be the main storage key
                    // to access its data
                    Err(InstructionError::InvalidArgument)?;
                }

                if let StorageContract::ValidatorStorage {
                    reward_validations, ..
                } = &mut storage_contract
                {
                    let claims_index = get_segment_from_entry(entry_height);
                    let _num_validations = count_valid_proofs(&reward_validations[claims_index]);
                    // TODO can't just create lamports out of thin air
                    // keyed_accounts[0].account.lamports += TOTAL_VALIDATOR_REWARDS * num_validations;
                    reward_validations.clear();
                } else if let StorageContract::ReplicatorStorage {
                    reward_validations, ..
                } = &mut storage_contract
                {
                    // if current tick height is a full segment away? then allow reward collection
                    // storage needs to move to tick heights too, until then this makes little sense
                    let current_index = get_segment_from_entry(tick_height);
                    let claims_index = get_segment_from_entry(entry_height);
                    if current_index <= claims_index || claims_index >= reward_validations.len() {
                        println!(
                            "current {:?}, claim {:?}, rewards {:?}",
                            current_index,
                            claims_index,
                            reward_validations.len()
                        );
                        return Err(InstructionError::InvalidArgument);
                    }
                    let _num_validations = count_valid_proofs(&reward_validations[claims_index]);
                    // TODO can't just create lamports out of thin air
                    // keyed_accounts[0].account.lamports += num_validations
                    //     * TOTAL_REPLICATOR_REWARDS
                    //     * (num_validations / reward_validations[claims_index].len() as u64);
                    reward_validations.clear();
                } else {
                    return Err(InstructionError::InvalidArgument);
                }
            }
            StorageInstruction::ProofValidation {
                entry_height: proof_entry_height,
                proofs,
            } => {
                if let StorageContract::ValidatorStorage {
                    entry_height: current_entry_height,
                    lockout_validations,
                    ..
                } = &mut storage_contract
                {
                    if keyed_accounts.len() == 1 {
                        // have to have at least 1 replicator to do any verification
                        Err(InstructionError::InvalidArgument)?;
                    }

                    if proof_entry_height >= *current_entry_height {
                        return Err(InstructionError::InvalidArgument);
                    }

                    let segment_index = get_segment_from_entry(proof_entry_height);
                    let mut previous_proofs = keyed_accounts[1..]
                        .iter_mut()
                        .filter_map(|account| {
                            read_contract(&account.account)
                                .ok()
                                .map(move |contract| match contract {
                                    StorageContract::ReplicatorStorage { proofs, .. } => {
                                        Some((&mut account.account, proofs[segment_index].clone()))
                                    }
                                    _ => None,
                                })
                        })
                        .flatten()
                        .collect::<Vec<_>>();

                    if previous_proofs.len() != proofs.len() {
                        // don't have all the accounts to validate the proofs against
                        return Err(InstructionError::InvalidArgument);
                    }

                    let mut valid_proofs: Vec<_> = proofs
                        .into_iter()
                        .enumerate()
                        .filter_map(|(i, entry)| {
                            if previous_proofs[i].1.signature != entry.proof.signature
                                || entry.status != ProofStatus::Valid
                            {
                                let _ = store_validation_result(
                                    &mut previous_proofs[i].0,
                                    segment_index,
                                    entry.status,
                                );
                                None
                            } else if store_validation_result(
                                &mut previous_proofs[i].0,
                                segment_index,
                                entry.status.clone(),
                            )
                            .is_ok()
                            {
                                Some(entry)
                            } else {
                                None
                            }
                        })
                        .collect();

                    // allow validators to store successful validations
                    lockout_validations[segment_index].append(&mut valid_proofs);
                } else {
                    return Err(InstructionError::InvalidArgument);
                }
            }
        }
        store_contract(&mut keyed_accounts[0].account, &storage_contract)?;
        Ok(())
    } else {
        info!("Invalid instruction data: {:?}", data);
        Err(InstructionError::InvalidInstructionData)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::id;
    use crate::storage_instruction;
    use crate::ENTRIES_PER_SEGMENT;
    use bincode::deserialize;
    use solana_runtime::bank::Bank;
    use solana_runtime::bank_client::BankClient;
    use solana_sdk::account::{create_keyed_accounts, Account};
    use solana_sdk::genesis_block::GenesisBlock;
    use solana_sdk::hash::{hash, Hash};
    use solana_sdk::instruction::Instruction;
    use solana_sdk::pubkey::Pubkey;
    use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
    use solana_sdk::system_instruction;

    fn test_instruction(
        ix: &Instruction,
        program_accounts: &mut [Account],
    ) -> Result<(), InstructionError> {
        let mut keyed_accounts: Vec<_> = ix
            .accounts
            .iter()
            .zip(program_accounts.iter_mut())
            .map(|(account_meta, account)| {
                KeyedAccount::new(&account_meta.pubkey, account_meta.is_signer, account)
            })
            .collect();

        let ret = process_instruction(&id(), &mut keyed_accounts, &ix.data, 42);
        info!("ret: {:?}", ret);
        ret
    }

    #[test]
    fn test_storage_tx() {
        let pubkey = Pubkey::new_rand();
        let mut accounts = [(pubkey, Account::default())];
        let mut keyed_accounts = create_keyed_accounts(&mut accounts);
        assert!(process_instruction(&id(), &mut keyed_accounts, &[], 42).is_err());
    }

    #[test]
    fn test_serialize_overflow() {
        let pubkey = Pubkey::new_rand();
        let mut keyed_accounts = Vec::new();
        let mut user_account = Account::default();
        keyed_accounts.push(KeyedAccount::new(&pubkey, true, &mut user_account));

        let ix = storage_instruction::advertise_recent_blockhash(
            &pubkey,
            Hash::default(),
            ENTRIES_PER_SEGMENT,
        );

        assert_eq!(
            process_instruction(&id(), &mut keyed_accounts, &ix.data, 42),
            Err(InstructionError::InvalidAccountData)
        );
    }

    #[test]
    fn test_invalid_accounts_len() {
        let pubkey = Pubkey::new_rand();
        let mut accounts = [Account::default()];

        let ix =
            storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default());
        assert!(test_instruction(&ix, &mut accounts).is_err());

        let mut accounts = [Account::default(), Account::default(), Account::default()];

        assert!(test_instruction(&ix, &mut accounts).is_err());
    }

    #[test]
    fn test_submit_mining_invalid_entry_height() {
        solana_logger::setup();
        let pubkey = Pubkey::new_rand();
        let mut accounts = [Account::default(), Account::default()];
        accounts[1].data.resize(16 * 1024, 0);

        let ix =
            storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default());

        // Haven't seen a transaction to roll over the epoch, so this should fail
        assert!(test_instruction(&ix, &mut accounts).is_err());
    }

    #[test]
    fn test_submit_mining_ok() {
        solana_logger::setup();
        let pubkey = Pubkey::new_rand();
        let mut accounts = [Account::default(), Account::default()];
        accounts[0].data.resize(16 * 1024, 0);

        let ix =
            storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default());

        test_instruction(&ix, &mut accounts).unwrap();
    }

    #[test]
    fn test_account_data() {
        solana_logger::setup();
        let mut account = Account::default();
        account.data.resize(4 * 1024, 0);
        let pubkey = &Pubkey::default();
        let mut keyed_account = KeyedAccount::new(&pubkey, false, &mut account);
        // pretend it's a validator op code
        let mut contract = read_contract(&keyed_account.account).unwrap();
        if let StorageContract::ValidatorStorage { .. } = contract {
            assert!(true)
        }
        if let StorageContract::ReplicatorStorage { .. } = &mut contract {
            panic!("this shouldn't work");
        }

        contract = StorageContract::ValidatorStorage {
            entry_height: 0,
            hash: Hash::default(),
            lockout_validations: vec![],
            reward_validations: vec![],
        };
        store_contract(&mut keyed_account.account, &contract).unwrap();
        if let StorageContract::ReplicatorStorage { .. } = contract {
            panic!("this shouldn't work");
        }
        contract = StorageContract::ReplicatorStorage {
            proofs: vec![],
            reward_validations: vec![],
        };
        store_contract(&mut keyed_account.account, &contract).unwrap();
        if let StorageContract::ValidatorStorage { .. } = contract {
            panic!("this shouldn't work");
        }
    }

    #[test]
    fn test_validate_mining() {
        solana_logger::setup();
        let (genesis_block, mint_keypair) = GenesisBlock::new(1000);
        let mint_pubkey = mint_keypair.pubkey();
        let replicator_keypair = Keypair::new();
        let replicator = replicator_keypair.pubkey();
        let validator_keypair = Keypair::new();
        let validator = validator_keypair.pubkey();

        let mut bank = Bank::new(&genesis_block);
        bank.add_instruction_processor(id(), process_instruction);
        let entry_height = 0;
        let bank_client = BankClient::new(&bank);

        let ix = system_instruction::create_account(&mint_pubkey, &validator, 10, 4 * 1042, &id());
        bank_client.process_instruction(&mint_keypair, ix).unwrap();

        let ix = system_instruction::create_account(&mint_pubkey, &replicator, 10, 4 * 1042, &id());
        bank_client.process_instruction(&mint_keypair, ix).unwrap();

        let ix = storage_instruction::advertise_recent_blockhash(
            &validator,
            Hash::default(),
            ENTRIES_PER_SEGMENT,
        );

        bank_client
            .process_instruction(&validator_keypair, ix)
            .unwrap();

        let ix = storage_instruction::mining_proof(
            &replicator,
            Hash::default(),
            entry_height,
            Signature::default(),
        );
        bank_client
            .process_instruction(&replicator_keypair, ix)
            .unwrap();

        let ix = storage_instruction::advertise_recent_blockhash(
            &validator,
            Hash::default(),
            ENTRIES_PER_SEGMENT * 2,
        );
        bank_client
            .process_instruction(&validator_keypair, ix)
            .unwrap();

        let ix = storage_instruction::proof_validation(
            &validator,
            entry_height,
            vec![CheckedProof {
                proof: Proof {
                    id: replicator,
                    signature: Signature::default(),
                    sha_state: Hash::default(),
                },
                status: ProofStatus::Valid,
            }],
        );
        bank_client
            .process_instruction(&validator_keypair, ix)
            .unwrap();

        let ix = storage_instruction::advertise_recent_blockhash(
            &validator,
            Hash::default(),
            ENTRIES_PER_SEGMENT * 3,
        );
        bank_client
            .process_instruction(&validator_keypair, ix)
            .unwrap();

        let ix = storage_instruction::reward_claim(&validator, entry_height);
        bank_client
            .process_instruction(&validator_keypair, ix)
            .unwrap();

        // TODO enable when rewards are working
        // assert_eq!(bank.get_balance(&validator), TOTAL_VALIDATOR_REWARDS);

        // tick the bank into the next storage epoch so that rewards can be claimed
        for _ in 0..=ENTRIES_PER_SEGMENT {
            bank.register_tick(&bank.last_blockhash());
        }

        let ix = storage_instruction::reward_claim(&replicator, entry_height);
        bank_client
            .process_instruction(&replicator_keypair, ix)
            .unwrap();

        // TODO enable when rewards are working
        // assert_eq!(bank.get_balance(&replicator), TOTAL_REPLICATOR_REWARDS);
    }

    fn get_storage_entry_height(bank: &Bank, account: &Pubkey) -> u64 {
        match bank.get_account(&account) {
            Some(storage_system_account) => {
                let contract = deserialize(&storage_system_account.data);
                if let Ok(contract) = contract {
                    match contract {
                        StorageContract::ValidatorStorage { entry_height, .. } => {
                            return entry_height;
                        }
                        _ => info!("error in reading entry_height"),
                    }
                }
            }
            None => {
                info!("error in reading entry_height");
            }
        }
        0
    }

    fn get_storage_blockhash(bank: &Bank, account: &Pubkey) -> Hash {
        if let Some(storage_system_account) = bank.get_account(&account) {
            let contract = deserialize(&storage_system_account.data);
            if let Ok(contract) = contract {
                match contract {
                    StorageContract::ValidatorStorage { hash, .. } => {
                        return hash;
                    }
                    _ => (),
                }
            }
        }
        Hash::default()
    }

    #[test]
    fn test_bank_storage() {
        let (genesis_block, mint_keypair) = GenesisBlock::new(1000);
        let mint_pubkey = mint_keypair.pubkey();
        let replicator_keypair = Keypair::new();
        let replicator_pubkey = replicator_keypair.pubkey();
        let validator_keypair = Keypair::new();
        let validator_pubkey = validator_keypair.pubkey();

        let mut bank = Bank::new(&genesis_block);
        bank.add_instruction_processor(id(), process_instruction);
        let bank_client = BankClient::new(&bank);

        let x = 42;
        let blockhash = genesis_block.hash();
        let x2 = x * 2;
        let storage_blockhash = hash(&[x2]);

        bank.register_tick(&blockhash);

        bank_client
            .transfer(10, &mint_keypair, &replicator_pubkey)
            .unwrap();

        let ix = system_instruction::create_account(
            &mint_pubkey,
            &replicator_pubkey,
            1,
            4 * 1024,
            &id(),
        );

        bank_client.process_instruction(&mint_keypair, ix).unwrap();

        let ix =
            system_instruction::create_account(&mint_pubkey, &validator_pubkey, 1, 4 * 1024, &id());

        bank_client.process_instruction(&mint_keypair, ix).unwrap();

        let ix = storage_instruction::advertise_recent_blockhash(
            &validator_pubkey,
            storage_blockhash,
            ENTRIES_PER_SEGMENT,
        );

        bank_client
            .process_instruction(&validator_keypair, ix)
            .unwrap();

        let entry_height = 0;
        let ix = storage_instruction::mining_proof(
            &replicator_pubkey,
            Hash::default(),
            entry_height,
            Signature::default(),
        );
        let _result = bank_client
            .process_instruction(&replicator_keypair, ix)
            .unwrap();

        assert_eq!(
            get_storage_entry_height(&bank, &validator_pubkey),
            ENTRIES_PER_SEGMENT
        );
        assert_eq!(
            get_storage_blockhash(&bank, &validator_pubkey),
            storage_blockhash
        );
    }

    /// check that uninitialized accounts are handled
    #[test]
    fn test_read_contract_with_default() {
        let mut account = Account::default();
        // no space allocated
        assert!(read_contract(&account).is_err());
        account.data.resize(4 * 1024, 0);
        let instruction = StorageInstruction::AdvertiseStorageRecentBlockhash {
            hash: Hash::default(),
            entry_height: 0,
        };
        read_contract_with_default(&instruction, &account).unwrap();
        let instruction = StorageInstruction::SubmitMiningProof {
            sha_state: Hash::default(),
            entry_height: 0,
            signature: Signature::default(),
        };
        read_contract_with_default(&instruction, &account).unwrap();
        let instruction = StorageInstruction::ProofValidation {
            entry_height: 0,
            proofs: vec![],
        };
        read_contract_with_default(&instruction, &account).unwrap();
        // Can't claim rewards on an uninitialized account
        let instruction = StorageInstruction::ClaimStorageReward { entry_height: 0 };
        assert!(read_contract_with_default(&instruction, &account).is_err());
    }
}