Make account shrink configurable #17544 (backport #17778) (#18013)

* Make account shrink configurable #17544 (#17778)

1. Added both options for measuring space usage using total accounts usage and for individual store shrink ratio using an enum. Validator CLI options: --accounts-shrink-optimize-total-space and --accounts-shrink-ratio
2. Added code for selecting candidates based on total usage in a separate function select_candidates_by_total_usage
3. Added unit tests for the new functions added
4. The default implementations is kept at 0.8 shrink ratio with --accounts-shrink-optimize-total-space set to true

Fixes #17544

(cherry picked from commit 269d995832)

# Conflicts:
#	core/tests/snapshots.rs
#	ledger/src/bank_forks_utils.rs
#	runtime/src/accounts_db.rs
#	runtime/src/snapshot_utils.rs

* fix some merge errors

Co-authored-by: Lijun Wang <83639177+lijunwangs@users.noreply.github.com>
Co-authored-by: Jeff Washington (jwash) <wash678@gmail.com>

validator/src/main.rs

@@ -39,6 +39,10 @@ use {
     solana_poh::poh_service,
     solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
     solana_runtime::{
+        accounts_db::{
+            AccountShrinkThreshold, DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE,
+            DEFAULT_ACCOUNTS_SHRINK_RATIO,
+        },
         accounts_index::{
             AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
         },
@@ -1011,6 +1015,9 @@ pub fn main() {
     let default_max_snapshot_to_retain = &DEFAULT_MAX_SNAPSHOTS_TO_RETAIN.to_string();
     let default_min_snapshot_download_speed = &DEFAULT_MIN_SNAPSHOT_DOWNLOAD_SPEED.to_string();
     let default_max_snapshot_download_abort = &MAX_SNAPSHOT_DOWNLOAD_ABORT.to_string();
+    let default_accounts_shrink_optimize_total_space =
+        &DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE.to_string();
+    let default_accounts_shrink_ratio = &DEFAULT_ACCOUNTS_SHRINK_RATIO.to_string();
 
     let matches = App::new(crate_name!()).about(crate_description!())
         .version(solana_version::version!())
@@ -1789,6 +1796,29 @@ pub fn main() {
                 .conflicts_with("no_accounts_db_caching")
                 .hidden(true)
         )
+        .arg(
+            Arg::with_name("accounts_shrink_optimize_total_space")
+                .long("accounts-shrink-optimize-total-space")
+                .takes_value(true)
+                .value_name("BOOLEAN")
+                .default_value(default_accounts_shrink_optimize_total_space)
+                .help("When this is set to true, the system will shrink the most \
+                       sparse accounts and when the overall shrink ratio is above \
+                       the specified accounts-shrink-ratio, the shrink will stop and \
+                       it will skip all other less sparse accounts."),
+        )
+        .arg(
+            Arg::with_name("accounts_shrink_ratio")
+                .long("accounts-shrink-ratio")
+                .takes_value(true)
+                .value_name("RATIO")
+                .default_value(default_accounts_shrink_ratio)
+                .help("Specifies the shrink ratio for the accounts to be shrunk. \
+                       The shrink ratio is defined as the ratio of the bytes alive over the  \
+                       total bytes used. If the account's shrink ratio is less than this ratio \
+                       it becomes a candidate for shrinking. The value must between 0. and 1.0 \
+                       inclusive."),
+        )
         .arg(
             Arg::with_name("no_duplicate_instance_check")
                 .long("no-duplicate-instance-check")
@@ -2087,6 +2117,23 @@ pub fn main() {
     let account_indexes = process_account_indexes(&matches);
 
     let restricted_repair_only_mode = matches.is_present("restricted_repair_only_mode");
+    let accounts_shrink_optimize_total_space =
+        value_t_or_exit!(matches, "accounts_shrink_optimize_total_space", bool);
+    let shrink_ratio = value_t_or_exit!(matches, "accounts_shrink_ratio", f64);
+    if !(0.0..=1.0).contains(&shrink_ratio) {
+        eprintln!(
+            "The specified account-shrink-ratio is invalid, it must be between 0. and 1.0 inclusive: {}",
+            shrink_ratio
+        );
+        exit(1);
+    }
+
+    let accounts_shrink_ratio = if accounts_shrink_optimize_total_space {
+        AccountShrinkThreshold::TotalSpace { shrink_ratio }
+    } else {
+        AccountShrinkThreshold::IndividalStore { shrink_ratio }
+    };
+
     let mut validator_config = ValidatorConfig {
         require_tower: matches.is_present("require_tower"),
         tower_path: value_t!(matches, "tower", PathBuf).ok(),
@@ -2189,6 +2236,7 @@ pub fn main() {
         accounts_db_use_index_hash_calculation: matches.is_present("accounts_db_index_hashing"),
         tpu_coalesce_ms,
         no_wait_for_vote_to_start_leader: matches.is_present("no_wait_for_vote_to_start_leader"),
+        accounts_shrink_ratio,
         ..ValidatorConfig::default()
     };