From 4b3d64ec9f00d5a0a3bd4cfdc013490269978b37 Mon Sep 17 00:00:00 2001 From: Stephen Akridge Date: Mon, 7 Jan 2019 17:08:03 -0800 Subject: [PATCH] Convert chacha_encrypt_file to work with db_ledger blobs directly --- src/chacha.rs | 122 +++++++++++++++++++++++++++++++++++---------- src/chacha_cuda.rs | 40 ++++++--------- src/ledger.rs | 2 +- src/replicator.rs | 15 +++--- 4 files changed, 120 insertions(+), 59 deletions(-) diff --git a/src/chacha.rs b/src/chacha.rs index 0aa18a0e35..1f247ee437 100644 --- a/src/chacha.rs +++ b/src/chacha.rs @@ -1,9 +1,11 @@ +use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT}; use std::fs::File; use std::io; -use std::io::Read; -use std::io::Write; -use std::io::{BufReader, BufWriter}; +use std::io::{BufWriter, Write}; use std::path::Path; +use std::sync::Arc; + +use crate::storage_stage::ENTRIES_PER_SEGMENT; pub const CHACHA_BLOCK_SIZE: usize = 64; pub const CHACHA_KEY_SIZE: usize = 32; @@ -31,27 +33,46 @@ pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mu } } -pub fn chacha_cbc_encrypt_file( - in_path: &Path, +pub fn chacha_cbc_encrypt_ledger( + db_ledger: &Arc, + slice: u64, out_path: &Path, ivec: &mut [u8; CHACHA_BLOCK_SIZE], ) -> io::Result<()> { - let mut in_file = BufReader::new(File::open(in_path).expect("Can't open ledger data file")); let mut out_file = BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file")); let mut buffer = [0; 4 * 1024]; let mut encrypted_buffer = [0; 4 * 1024]; let key = [0; CHACHA_KEY_SIZE]; + let mut total_entries = 0; + let mut entry = slice; - while let Ok(size) = in_file.read(&mut buffer) { - debug!("read {} bytes", size); - if size == 0 { - break; - } - chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec); - if let Err(res) = out_file.write(&encrypted_buffer[..size]) { - println!("Error writing file! {:?}", res); - return Err(res); + loop { + match db_ledger.get_blob_bytes( + entry, + ENTRIES_PER_SEGMENT - total_entries, + &mut buffer, + DEFAULT_SLOT_HEIGHT, + ) { + Ok((num_entries, entry_len)) => { + debug!("read {} bytes", entry_len); + let size = entry_len as usize; + if size == 0 { + break; + } + chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec); + if let Err(res) = out_file.write(&encrypted_buffer[..size]) { + println!("Error writing file! {:?}", res); + return Err(res); + } + + total_entries += num_entries; + entry += num_entries; + } + Err(e) => { + info!("Error encrypting file: {:?}", e); + break; + } } } Ok(()) @@ -59,34 +80,81 @@ pub fn chacha_cbc_encrypt_file( #[cfg(test)] mod tests { - use crate::chacha::chacha_cbc_encrypt_file; + use crate::chacha::chacha_cbc_encrypt_ledger; + use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT}; + use crate::entry::Entry; + use crate::ledger::get_tmp_ledger_path; + use ring::signature::Ed25519KeyPair; + use solana_sdk::budget_transaction::BudgetTransaction; + use solana_sdk::hash::{hash, Hash, Hasher}; + use solana_sdk::signature::KeypairUtil; + use solana_sdk::transaction::Transaction; use std::fs::remove_file; use std::fs::File; use std::io::Read; - use std::io::Write; use std::path::Path; + use std::sync::Arc; + use untrusted::Input; + + fn make_tiny_deterministic_test_entries(num: usize) -> Vec { + let zero = Hash::default(); + let one = hash(&zero.as_ref()); + let pkcs = [ + 48, 83, 2, 1, 1, 48, 5, 6, 3, 43, 101, 112, 4, 34, 4, 32, 109, 148, 235, 20, 97, 127, + 43, 194, 109, 43, 121, 76, 54, 38, 234, 14, 108, 68, 209, 227, 137, 191, 167, 144, 177, + 174, 57, 182, 79, 198, 196, 93, 161, 35, 3, 33, 0, 116, 121, 255, 78, 31, 95, 179, 172, + 30, 125, 206, 87, 88, 78, 46, 145, 25, 154, 161, 252, 3, 58, 235, 116, 39, 148, 193, + 150, 111, 61, 20, 226, + ]; + let keypair = Ed25519KeyPair::from_pkcs8(Input::from(&pkcs)).unwrap(); + + let mut id = one; + let mut num_hashes = 0; + (0..num) + .map(|_| { + Entry::new_mut( + &mut id, + &mut num_hashes, + vec![Transaction::budget_new_signature( + &keypair, + keypair.pubkey(), + keypair.pubkey(), + one, + )], + ) + }) + .collect() + } #[test] - fn test_encrypt_file() { - let in_path = Path::new("test_chacha_encrypt_file_input.txt"); + fn test_encrypt_ledger() { + solana_logger::setup(); + let ledger_dir = "chacha_test_encrypt_file"; + let ledger_path = get_tmp_ledger_path(ledger_dir); + let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap()); let out_path = Path::new("test_chacha_encrypt_file_output.txt.enc"); - { - let mut in_file = File::create(in_path).unwrap(); - in_file.write("123456foobar".as_bytes()).unwrap(); - } + + let entries = make_tiny_deterministic_test_entries(32); + db_ledger + .write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries) + .unwrap(); + let mut key = hex!( "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234" ); - assert!(chacha_cbc_encrypt_file(in_path, out_path, &mut key).is_ok()); + assert!(chacha_cbc_encrypt_ledger(&db_ledger, 0, out_path, &mut key).is_ok()); let mut out_file = File::open(out_path).unwrap(); let mut buf = vec![]; let size = out_file.read_to_end(&mut buf).unwrap(); + let mut hasher = Hasher::default(); + hasher.hash(&buf[..size]); assert_eq!( - buf[..size], - [66, 54, 56, 212, 142, 110, 105, 158, 116, 82, 120, 53] + hasher.result(), + Hash::new(&hex!( + "58433c941060af56b72bfeaca161f19ed6df531efb28961dc6b83f53fbf66ffe" + )), ); - remove_file(in_path).unwrap(); remove_file(out_path).unwrap(); } } diff --git a/src/chacha_cuda.rs b/src/chacha_cuda.rs index f44841aedd..e6ae91ba72 100644 --- a/src/chacha_cuda.rs +++ b/src/chacha_cuda.rs @@ -1,5 +1,5 @@ use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE}; -use crate::db_ledger::DbLedger; +use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT}; use crate::sigverify::{ chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state, }; @@ -44,7 +44,12 @@ pub fn chacha_cbc_encrypt_file_many_keys( chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32); } loop { - match db_ledger.get_entries_bytes(entry, ENTRIES_PER_SEGMENT - total_entries, &mut buffer) { + match db_ledger.get_blob_bytes( + entry, + ENTRIES_PER_SEGMENT - total_entries, + &mut buffer, + DEFAULT_SLOT_HEIGHT, + ) { Ok((num_entries, entry_len)) => { info!( "encrypting slice: {} num_entries: {} entry_len: {}", @@ -101,12 +106,10 @@ pub fn chacha_cbc_encrypt_file_many_keys( #[cfg(test)] mod tests { - use crate::chacha::chacha_cbc_encrypt_file; + use crate::chacha::chacha_cbc_encrypt_ledger; use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys; use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT}; - use crate::ledger::{ - get_tmp_ledger_path, make_tiny_test_entries, LedgerWriter, LEDGER_DATA_FILE, - }; + use crate::ledger::{get_tmp_ledger_path, make_tiny_test_entries, LedgerWriter}; use crate::replicator::sample_file; use solana_sdk::hash::Hash; use std::fs::{remove_dir_all, remove_file}; @@ -114,7 +117,6 @@ mod tests { use std::sync::Arc; #[test] - #[ignore] fn test_encrypt_file_many_keys_single() { solana_logger::setup(); @@ -125,7 +127,7 @@ mod tests { let mut writer = LedgerWriter::open(&ledger_path, true).unwrap(); writer.write_entries(&entries).unwrap(); } - let db_ledger = DbLedger::open(&ledger_path).unwrap(); + let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap()); db_ledger .write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries) .unwrap(); @@ -139,18 +141,12 @@ mod tests { ); let mut cpu_iv = ivecs.clone(); - assert!(chacha_cbc_encrypt_file( - &Path::new(&ledger_path).join(LEDGER_DATA_FILE), - out_path, - &mut cpu_iv, - ) - .is_ok()); + assert!(chacha_cbc_encrypt_ledger(&db_ledger, 0, out_path, &mut cpu_iv,).is_ok()); let ref_hash = sample_file(&out_path, &samples).unwrap(); let hashes = - chacha_cbc_encrypt_file_many_keys(&Arc::new(db_ledger), 0, &mut ivecs, &samples) - .unwrap(); + chacha_cbc_encrypt_file_many_keys(&db_ledger, 0, &mut ivecs, &samples).unwrap(); assert_eq!(hashes[0], ref_hash); @@ -170,7 +166,7 @@ mod tests { let mut writer = LedgerWriter::open(&ledger_path, true).unwrap(); writer.write_entries(&entries).unwrap(); } - let db_ledger = DbLedger::open(&ledger_path).unwrap(); + let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap()); db_ledger .write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries) .unwrap(); @@ -187,12 +183,7 @@ mod tests { ); ivec[0] = i; ivecs.extend(ivec.clone().iter()); - assert!(chacha_cbc_encrypt_file( - &Path::new(&ledger_path).join(LEDGER_DATA_FILE), - out_path, - &mut ivec, - ) - .is_ok()); + assert!(chacha_cbc_encrypt_ledger(&db_ledger.clone(), 0, out_path, &mut ivec,).is_ok()); ref_hashes.push(sample_file(&out_path, &samples).unwrap()); info!( @@ -204,8 +195,7 @@ mod tests { } let hashes = - chacha_cbc_encrypt_file_many_keys(&Arc::new(db_ledger), 0, &mut ivecs, &samples) - .unwrap(); + chacha_cbc_encrypt_file_many_keys(&db_ledger, 0, &mut ivecs, &samples).unwrap(); assert_eq!(hashes, ref_hashes); diff --git a/src/ledger.rs b/src/ledger.rs index f0209b5454..2d76c8b080 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -69,7 +69,7 @@ struct LedgerWindow { data: BufReader, } -pub const LEDGER_DATA_FILE: &str = "data"; +const LEDGER_DATA_FILE: &str = "data"; const LEDGER_INDEX_FILE: &str = "index"; // use a CONST because there's a cast, and we don't want "sizeof:: as u64"... diff --git a/src/replicator.rs b/src/replicator.rs index afde66c8ad..b387fe80ad 100644 --- a/src/replicator.rs +++ b/src/replicator.rs @@ -1,12 +1,11 @@ use crate::blob_fetch_stage::BlobFetchStage; #[cfg(feature = "chacha")] -use crate::chacha::{chacha_cbc_encrypt_file, CHACHA_BLOCK_SIZE}; +use crate::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE}; use crate::client::mk_client; use crate::cluster_info::{ClusterInfo, Node, NodeInfo}; use crate::db_ledger::DbLedger; use crate::gossip_service::GossipService; use crate::leader_scheduler::LeaderScheduler; -use crate::ledger::LEDGER_DATA_FILE; use crate::result::Result; use crate::rpc_request::{RpcClient, RpcRequest}; use crate::service::Service; @@ -185,7 +184,7 @@ impl Replicator { let (retransmit_sender, retransmit_receiver) = channel(); let t_window = window_service( - db_ledger, + db_ledger.clone(), cluster_info.clone(), 0, entry_height, @@ -243,14 +242,18 @@ impl Replicator { info!("Done downloading ledger at {}", ledger_path.unwrap()); let ledger_path = Path::new(ledger_path.unwrap()); - let ledger_data_file_encrypted = ledger_path.join(format!("{}.enc", LEDGER_DATA_FILE)); + let ledger_data_file_encrypted = ledger_path.join("ledger.enc"); #[cfg(feature = "chacha")] { - let ledger_data_file = ledger_path.join(LEDGER_DATA_FILE); let mut ivec = [0u8; CHACHA_BLOCK_SIZE]; ivec[0..4].copy_from_slice(&[2, 3, 4, 5]); - chacha_cbc_encrypt_file(&ledger_data_file, &ledger_data_file_encrypted, &mut ivec)?; + chacha_cbc_encrypt_ledger( + &db_ledger, + entry_height, + &ledger_data_file_encrypted, + &mut ivec, + )?; } info!("Done encrypting the ledger");