Avoid full-range compactions with periodic filtered b.g. ones (#16697)

* Update rocksdb to v0.16.0

* Promote the infrequent and important log to info!

* Force background compaction by ttl without manual compaction

* Fix test

* Support no compaction mode in test_ledger_cleanup_compaction

* Fix comment

* Make compaction_interval customizable

* Avoid major compaction with periodic filtering...

* Adress lazy_static, special cfs and range check

* Clean up a bit and add comment

* Add comment

* More comments...

* Config code cleanup

* Add comment

* Use .conflicts_with()

* Nullify unneeded delete_range ops for special CFs

* Some clean ups

* Clarify the locking intention

* Ensure special CFs' consistency with PurgeType::CompactionFilter

* Fix comment

* Fix bad copy paste

* Fix various types...

* Don't use tuples

* Add a unit test for compaction_filter

* Fix typo...

* Remove flag and just use new behavior always

* Fix wrong condition negation...

* Doc. about no set_last_purged_slot in purge_slots

* Write a test and fix off-by-one bug....

* Apply suggestions from code review

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>

* Follow up to github review suggestions

* Fix line-wrapping

* Fix conflict

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>

core/src/ledger_cleanup_service.rs

@@ -207,11 +207,25 @@ impl LedgerCleanupService {
                     );
 
                     let mut purge_time = Measure::start("purge_slots");
+
                     blockstore.purge_slots(
                         purge_first_slot,
                         lowest_cleanup_slot,
-                        PurgeType::PrimaryIndex,
+                        PurgeType::CompactionFilter,
                     );
+                    // Update only after purge operation.
+                    // Safety: This value can be used by compaction_filters shared via Arc<AtomicU64>.
+                    // Compactions are async and run as a multi-threaded background job. However, this
+                    // shouldn't cause consistency issues for iterators and getters because we have
+                    // already expired all affected keys (older than or equal to lowest_cleanup_slot)
+                    // by the above `purge_slots`. According to the general RocksDB design where SST
+                    // files are immutable, even running iterators aren't affected; the database grabs
+                    // a snapshot of the live set of sst files at iterator's creation.
+                    // Also, we passed the PurgeType::CompactionFilter, meaning no delete_range for
+                    // transaction_status and address_signatures CFs. These are fine because they
+                    // don't require strong consistent view for their operation.
+                    blockstore.set_max_expired_slot(lowest_cleanup_slot);
+
                     purge_time.stop();
                     info!("{}", purge_time);
 

core/src/validator.rs

@@ -1636,9 +1636,11 @@ mod tests {
             }
             drop(blockstore);
 
+            // this purges and compacts all slots greater than or equal to 5
             backup_and_clear_blockstore(&blockstore_path, 5, 2);
 
             let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            // assert that slots less than 5 aren't affected
             assert!(blockstore.meta(4).unwrap().unwrap().next_slots.is_empty());
             for i in 5..10 {
                 assert!(blockstore

core/tests/ledger_cleanup.rs

@@ -39,6 +39,8 @@ mod tests {
         pub cleanup_blockstore: bool,
         pub emit_cpu_info: bool,
         pub assert_compaction: bool,
+        pub compaction_interval: Option<u64>,
+        pub no_compaction: bool,
     }
 
     #[derive(Clone, Copy, Debug)]
@@ -154,6 +156,11 @@ mod tests {
         let emit_cpu_info = read_env("EMIT_CPU_INFO", true);
         // set default to `true` once compaction is merged
         let assert_compaction = read_env("ASSERT_COMPACTION", false);
+        let compaction_interval = match read_env("COMPACTION_INTERVAL", 0) {
+            maybe_zero if maybe_zero == 0 => None,
+            non_zero => Some(non_zero),
+        };
+        let no_compaction = read_env("NO_COMPACTION", false);
 
         BenchmarkConfig {
             benchmark_slots,
@@ -166,6 +173,8 @@ mod tests {
             cleanup_blockstore,
             emit_cpu_info,
             assert_compaction,
+            compaction_interval,
+            no_compaction,
         }
     }
 
@@ -211,8 +220,13 @@ mod tests {
     fn test_ledger_cleanup_compaction() {
         solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
-        let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap());
+        let mut blockstore = Blockstore::open(&blockstore_path).unwrap();
         let config = get_benchmark_config();
+        if config.no_compaction {
+            blockstore.set_no_compaction(true);
+        }
+        let blockstore = Arc::new(blockstore);
+
         eprintln!("BENCHMARK CONFIG: {:?}", config);
         eprintln!("LEDGER_PATH: {:?}", &blockstore_path);
 
@@ -223,6 +237,8 @@ mod tests {
         let stop_size_bytes = config.stop_size_bytes;
         let stop_size_iterations = config.stop_size_iterations;
         let pre_generate_data = config.pre_generate_data;
+        let compaction_interval = config.compaction_interval;
+
         let batches = benchmark_slots / batch_size;
 
         let (sender, receiver) = channel();
@@ -232,7 +248,7 @@ mod tests {
             blockstore.clone(),
             max_ledger_shreds,
             &exit,
-            None,
+            compaction_interval,
             None,
         );