Measure heap usage while processing the ledger at validator startup (bp #7667) (#7669)

automerge

core/Cargo.toml

@@ -71,10 +71,6 @@ untrusted = "0.7.0"
 solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.21.6" }
 reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
 
-[target."cfg(unix)".dependencies]
-jemallocator = "0.3.2"
-jemalloc-ctl = "0.3.2"
-
 [dev-dependencies]
 hex-literal = "0.2.1"
 matches = "0.1.6"

core/src/banking_stage.rs

@@ -7,7 +7,6 @@ use crate::{
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
     poh_service::PohService,
     result::{Error, Result},
-    thread_mem_usage,
 };
 use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError};
 use itertools::Itertools;
@@ -17,7 +16,7 @@ use solana_ledger::{
     entry::hash_transactions,
     leader_schedule_cache::LeaderScheduleCache,
 };
-use solana_measure::measure::Measure;
+use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn};
 use solana_perf::{cuda_runtime::PinnedVec, perf_libs};
 use solana_runtime::{accounts_db::ErrorCounters, bank::Bank, transaction_batch::TransactionBatch};

core/src/cluster_info.rs

@@ -24,7 +24,6 @@ use crate::{
     repair_service::RepairType,
     result::{Error, Result},
     sendmmsg::{multicast, send_mmsg},
-    thread_mem_usage,
     weighted_shuffle::{weighted_best, weighted_shuffle},
 };
 use bincode::{serialize, serialized_size};
@@ -32,6 +31,7 @@ use core::cmp;
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 use solana_ledger::{bank_forks::BankForks, blocktree::Blocktree, staking_utils};
+use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error};
 use solana_net_utils::{
     bind_common, bind_common_in_range, bind_in_range, find_available_port_in_range,

core/src/fetch_stage.rs

@@ -5,7 +5,7 @@ use crate::packet::PacketsRecycler;
 use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
 use crate::streamer::{self, PacketReceiver, PacketSender};
-use crate::thread_mem_usage;
+use solana_measure::thread_mem_usage;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
 use solana_perf::recycler::Recycler;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;

core/src/lib.rs

@@ -12,7 +12,6 @@ pub mod chacha_cuda;
 pub mod cluster_info_vote_listener;
 pub mod commitment;
 pub mod shred_fetch_stage;
-pub mod thread_mem_usage;
 #[macro_use]
 pub mod contact_info;
 pub mod archiver;
@@ -84,10 +83,3 @@ extern crate solana_metrics;
 #[cfg(test)]
 #[macro_use]
 extern crate matches;
-
-#[cfg(unix)]
-extern crate jemallocator;
-
-#[cfg(unix)]
-#[global_allocator]
-static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

core/src/replay_stage.rs

@@ -7,7 +7,6 @@ use crate::{
     poh_recorder::PohRecorder,
     result::{Error, Result},
     rpc_subscriptions::RpcSubscriptions,
-    thread_mem_usage,
 };
 use solana_ledger::{
     bank_forks::BankForks,
@@ -18,7 +17,7 @@ use solana_ledger::{
     leader_schedule_cache::LeaderScheduleCache,
     snapshot_package::SnapshotPackageSender,
 };
-use solana_measure::measure::Measure;
+use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::{

core/src/streamer.rs

@@ -4,7 +4,7 @@
 use crate::packet::{self, send_to, Packets, PacketsRecycler, PACKETS_PER_BATCH};
 use crate::recvmmsg::NUM_RCVMMSGS;
 use crate::result::{Error, Result};
-use crate::thread_mem_usage;
+use solana_measure::thread_mem_usage;
 use solana_sdk::timing::duration_as_ms;
 use std::net::UdpSocket;
 use std::sync::atomic::{AtomicBool, Ordering};

core/src/thread_mem_usage.rs

@@ -1,39 +0,0 @@
-#[cfg(unix)]
-use jemalloc_ctl::thread;
-
-pub fn datapoint(_name: &'static str) {
-    #[cfg(unix)]
-    {
-        let allocated = thread::allocatedp::mib().unwrap();
-        let allocated = allocated.read().unwrap();
-        let mem = allocated.get();
-        solana_metrics::datapoint_debug!("thread-memory", (_name, mem as i64, i64));
-    }
-}
-
-pub struct Allocatedp {
-    #[cfg(unix)]
-    allocated: thread::ThreadLocal<u64>,
-}
-
-impl Allocatedp {
-    pub fn default() -> Self {
-        #[cfg(unix)]
-        {
-            let allocated = thread::allocatedp::mib().unwrap();
-            let allocated = allocated.read().unwrap();
-            Self { allocated }
-        }
-        #[cfg(not(unix))]
-        Self {}
-    }
-
-    pub fn get(&self) -> u64 {
-        #[cfg(unix)]
-        {
-            self.allocated.get()
-        }
-        #[cfg(not(unix))]
-        0
-    }
-}