Convert chacha_encrypt_file to work with db_ledger blobs directly

src/chacha.rs

@@ -1,9 +1,11 @@
+use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
 use std::fs::File;
 use std::io;
-use std::io::Read;
+use std::io::{BufWriter, Write};
-use std::io::Write;
-use std::io::{BufReader, BufWriter};
 use std::path::Path;
+use std::sync::Arc;
+
+use crate::storage_stage::ENTRIES_PER_SEGMENT;
 
 pub const CHACHA_BLOCK_SIZE: usize = 64;
 pub const CHACHA_KEY_SIZE: usize = 32;
@@ -31,27 +33,46 @@ pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mu
     }
 }
 
-pub fn chacha_cbc_encrypt_file(
+pub fn chacha_cbc_encrypt_ledger(
-    in_path: &Path,
+    db_ledger: &Arc<DbLedger>,
+    slice: u64,
     out_path: &Path,
     ivec: &mut [u8; CHACHA_BLOCK_SIZE],
 ) -> io::Result<()> {
-    let mut in_file = BufReader::new(File::open(in_path).expect("Can't open ledger data file"));
     let mut out_file =
         BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file"));
     let mut buffer = [0; 4 * 1024];
     let mut encrypted_buffer = [0; 4 * 1024];
     let key = [0; CHACHA_KEY_SIZE];
+    let mut total_entries = 0;
+    let mut entry = slice;
 
-    while let Ok(size) = in_file.read(&mut buffer) {
+    loop {
-        debug!("read {} bytes", size);
+        match db_ledger.get_blob_bytes(
-        if size == 0 {
+            entry,
-            break;
+            ENTRIES_PER_SEGMENT - total_entries,
-        }
+            &mut buffer,
-        chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
+            DEFAULT_SLOT_HEIGHT,
-        if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
+        ) {
-            println!("Error writing file! {:?}", res);
+            Ok((num_entries, entry_len)) => {
-            return Err(res);
+                debug!("read {} bytes", entry_len);
+                let size = entry_len as usize;
+                if size == 0 {
+                    break;
+                }
+                chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
+                if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
+                    println!("Error writing file! {:?}", res);
+                    return Err(res);
+                }
+
+                total_entries += num_entries;
+                entry += num_entries;
+            }
+            Err(e) => {
+                info!("Error encrypting file: {:?}", e);
+                break;
+            }
         }
     }
     Ok(())
@@ -59,34 +80,81 @@ pub fn chacha_cbc_encrypt_file(
 
 #[cfg(test)]
 mod tests {
-    use crate::chacha::chacha_cbc_encrypt_file;
+    use crate::chacha::chacha_cbc_encrypt_ledger;
+    use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
+    use crate::entry::Entry;
+    use crate::ledger::get_tmp_ledger_path;
+    use ring::signature::Ed25519KeyPair;
+    use solana_sdk::budget_transaction::BudgetTransaction;
+    use solana_sdk::hash::{hash, Hash, Hasher};
+    use solana_sdk::signature::KeypairUtil;
+    use solana_sdk::transaction::Transaction;
     use std::fs::remove_file;
     use std::fs::File;
     use std::io::Read;
-    use std::io::Write;
     use std::path::Path;
+    use std::sync::Arc;
+    use untrusted::Input;
+
+    fn make_tiny_deterministic_test_entries(num: usize) -> Vec<Entry> {
+        let zero = Hash::default();
+        let one = hash(&zero.as_ref());
+        let pkcs = [
+            48, 83, 2, 1, 1, 48, 5, 6, 3, 43, 101, 112, 4, 34, 4, 32, 109, 148, 235, 20, 97, 127,
+            43, 194, 109, 43, 121, 76, 54, 38, 234, 14, 108, 68, 209, 227, 137, 191, 167, 144, 177,
+            174, 57, 182, 79, 198, 196, 93, 161, 35, 3, 33, 0, 116, 121, 255, 78, 31, 95, 179, 172,
+            30, 125, 206, 87, 88, 78, 46, 145, 25, 154, 161, 252, 3, 58, 235, 116, 39, 148, 193,
+            150, 111, 61, 20, 226,
+        ];
+        let keypair = Ed25519KeyPair::from_pkcs8(Input::from(&pkcs)).unwrap();
+
+        let mut id = one;
+        let mut num_hashes = 0;
+        (0..num)
+            .map(|_| {
+                Entry::new_mut(
+                    &mut id,
+                    &mut num_hashes,
+                    vec![Transaction::budget_new_signature(
+                        &keypair,
+                        keypair.pubkey(),
+                        keypair.pubkey(),
+                        one,
+                    )],
+                )
+            })
+            .collect()
+    }
 
     #[test]
-    fn test_encrypt_file() {
+    fn test_encrypt_ledger() {
-        let in_path = Path::new("test_chacha_encrypt_file_input.txt");
+        solana_logger::setup();
+        let ledger_dir = "chacha_test_encrypt_file";
+        let ledger_path = get_tmp_ledger_path(ledger_dir);
+        let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap());
         let out_path = Path::new("test_chacha_encrypt_file_output.txt.enc");
-        {
+
-            let mut in_file = File::create(in_path).unwrap();
+        let entries = make_tiny_deterministic_test_entries(32);
-            in_file.write("123456foobar".as_bytes()).unwrap();
+        db_ledger
-        }
+            .write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries)
+            .unwrap();
+
         let mut key = hex!(
             "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
                             abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
         );
-        assert!(chacha_cbc_encrypt_file(in_path, out_path, &mut key).is_ok());
+        assert!(chacha_cbc_encrypt_ledger(&db_ledger, 0, out_path, &mut key).is_ok());
         let mut out_file = File::open(out_path).unwrap();
         let mut buf = vec![];
         let size = out_file.read_to_end(&mut buf).unwrap();
+        let mut hasher = Hasher::default();
+        hasher.hash(&buf[..size]);
         assert_eq!(
-            buf[..size],
+            hasher.result(),
-            [66, 54, 56, 212, 142, 110, 105, 158, 116, 82, 120, 53]
+            Hash::new(&hex!(
+                "58433c941060af56b72bfeaca161f19ed6df531efb28961dc6b83f53fbf66ffe"
+            )),
         );
-        remove_file(in_path).unwrap();
         remove_file(out_path).unwrap();
     }
 }

src/chacha_cuda.rs

@@ -1,5 +1,5 @@
 use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
-use crate::db_ledger::DbLedger;
+use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
 use crate::sigverify::{
     chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state,
 };
@@ -44,7 +44,12 @@ pub fn chacha_cbc_encrypt_file_many_keys(
         chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32);
     }
     loop {
-        match db_ledger.get_entries_bytes(entry, ENTRIES_PER_SEGMENT - total_entries, &mut buffer) {
+        match db_ledger.get_blob_bytes(
+            entry,
+            ENTRIES_PER_SEGMENT - total_entries,
+            &mut buffer,
+            DEFAULT_SLOT_HEIGHT,
+        ) {
             Ok((num_entries, entry_len)) => {
                 info!(
                     "encrypting slice: {} num_entries: {} entry_len: {}",
@@ -101,12 +106,10 @@ pub fn chacha_cbc_encrypt_file_many_keys(
 
 #[cfg(test)]
 mod tests {
-    use crate::chacha::chacha_cbc_encrypt_file;
+    use crate::chacha::chacha_cbc_encrypt_ledger;
     use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
     use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
-    use crate::ledger::{
+    use crate::ledger::{get_tmp_ledger_path, make_tiny_test_entries, LedgerWriter};
-        get_tmp_ledger_path, make_tiny_test_entries, LedgerWriter, LEDGER_DATA_FILE,
-    };
     use crate::replicator::sample_file;
     use solana_sdk::hash::Hash;
     use std::fs::{remove_dir_all, remove_file};
@@ -114,7 +117,6 @@ mod tests {
     use std::sync::Arc;
 
     #[test]
-    #[ignore]
     fn test_encrypt_file_many_keys_single() {
         solana_logger::setup();
 
@@ -125,7 +127,7 @@ mod tests {
             let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
             writer.write_entries(&entries).unwrap();
         }
-        let db_ledger = DbLedger::open(&ledger_path).unwrap();
+        let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap());
         db_ledger
             .write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries)
             .unwrap();
@@ -139,18 +141,12 @@ mod tests {
         );
 
         let mut cpu_iv = ivecs.clone();
-        assert!(chacha_cbc_encrypt_file(
+        assert!(chacha_cbc_encrypt_ledger(&db_ledger, 0, out_path, &mut cpu_iv,).is_ok());
-            &Path::new(&ledger_path).join(LEDGER_DATA_FILE),
-            out_path,
-            &mut cpu_iv,
-        )
-        .is_ok());
 
         let ref_hash = sample_file(&out_path, &samples).unwrap();
 
         let hashes =
-            chacha_cbc_encrypt_file_many_keys(&Arc::new(db_ledger), 0, &mut ivecs, &samples)
+            chacha_cbc_encrypt_file_many_keys(&db_ledger, 0, &mut ivecs, &samples).unwrap();
-                .unwrap();
 
         assert_eq!(hashes[0], ref_hash);
 
@@ -170,7 +166,7 @@ mod tests {
             let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
             writer.write_entries(&entries).unwrap();
         }
-        let db_ledger = DbLedger::open(&ledger_path).unwrap();
+        let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap());
         db_ledger
             .write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries)
             .unwrap();
@@ -187,12 +183,7 @@ mod tests {
             );
             ivec[0] = i;
             ivecs.extend(ivec.clone().iter());
-            assert!(chacha_cbc_encrypt_file(
+            assert!(chacha_cbc_encrypt_ledger(&db_ledger.clone(), 0, out_path, &mut ivec,).is_ok());
-                &Path::new(&ledger_path).join(LEDGER_DATA_FILE),
-                out_path,
-                &mut ivec,
-            )
-            .is_ok());
 
             ref_hashes.push(sample_file(&out_path, &samples).unwrap());
             info!(
@@ -204,8 +195,7 @@ mod tests {
         }
 
         let hashes =
-            chacha_cbc_encrypt_file_many_keys(&Arc::new(db_ledger), 0, &mut ivecs, &samples)
+            chacha_cbc_encrypt_file_many_keys(&db_ledger, 0, &mut ivecs, &samples).unwrap();
-                .unwrap();
 
         assert_eq!(hashes, ref_hashes);
 

src/ledger.rs

@@ -69,7 +69,7 @@ struct LedgerWindow {
     data: BufReader<File>,
 }
 
-pub const LEDGER_DATA_FILE: &str = "data";
+const LEDGER_DATA_FILE: &str = "data";
 const LEDGER_INDEX_FILE: &str = "index";
 
 // use a CONST because there's a cast, and we don't want "sizeof::<u64> as u64"...

src/replicator.rs

@@ -1,12 +1,11 @@
 use crate::blob_fetch_stage::BlobFetchStage;
 #[cfg(feature = "chacha")]
-use crate::chacha::{chacha_cbc_encrypt_file, CHACHA_BLOCK_SIZE};
+use crate::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
 use crate::client::mk_client;
 use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
 use crate::db_ledger::DbLedger;
 use crate::gossip_service::GossipService;
 use crate::leader_scheduler::LeaderScheduler;
-use crate::ledger::LEDGER_DATA_FILE;
 use crate::result::Result;
 use crate::rpc_request::{RpcClient, RpcRequest};
 use crate::service::Service;
@@ -185,7 +184,7 @@ impl Replicator {
         let (retransmit_sender, retransmit_receiver) = channel();
 
         let t_window = window_service(
-            db_ledger,
+            db_ledger.clone(),
             cluster_info.clone(),
             0,
             entry_height,
@@ -243,14 +242,18 @@ impl Replicator {
         info!("Done downloading ledger at {}", ledger_path.unwrap());
 
         let ledger_path = Path::new(ledger_path.unwrap());
-        let ledger_data_file_encrypted = ledger_path.join(format!("{}.enc", LEDGER_DATA_FILE));
+        let ledger_data_file_encrypted = ledger_path.join("ledger.enc");
         #[cfg(feature = "chacha")]
         {
-            let ledger_data_file = ledger_path.join(LEDGER_DATA_FILE);
             let mut ivec = [0u8; CHACHA_BLOCK_SIZE];
             ivec[0..4].copy_from_slice(&[2, 3, 4, 5]);
 
-            chacha_cbc_encrypt_file(&ledger_data_file, &ledger_data_file_encrypted, &mut ivec)?;
+            chacha_cbc_encrypt_ledger(
+                &db_ledger,
+                entry_height,
+                &ledger_data_file_encrypted,
+                &mut ivec,
+            )?;
         }
 
         info!("Done encrypting the ledger");