Measure heap usage while processing the ledger at validator startup (bp #7667) (#7670)

automerge

measure/Cargo.toml

@@ -11,4 +11,10 @@ license = "Apache-2.0"
 edition = "2018"
 
 [dependencies]
+log = "0.4.8"
 solana-sdk = { path = "../sdk", version = "0.22.1" }
+solana-metrics = { path = "../metrics", version = "0.22.1" }
+
+[target."cfg(unix)".dependencies]
+jemallocator = "0.3.2"
+jemalloc-ctl = "0.3.2"

measure/src/lib.rs

@@ -1 +1,9 @@
 pub mod measure;
+pub mod thread_mem_usage;
+
+#[cfg(unix)]
+extern crate jemallocator;
+
+#[cfg(unix)]
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

measure/src/thread_mem_usage.rs

@@ -0,0 +1,45 @@
+#[cfg(unix)]
+use jemalloc_ctl::thread;
+
+pub fn datapoint(_name: &'static str) {
+    #[cfg(unix)]
+    {
+        let allocated = thread::allocatedp::mib().unwrap();
+        let allocated = allocated.read().unwrap();
+        let mem = allocated.get();
+        solana_metrics::datapoint_debug!("thread-memory", (_name, mem as i64, i64));
+    }
+}
+
+pub struct Allocatedp {
+    #[cfg(unix)]
+    allocated: thread::ThreadLocal<u64>,
+}
+
+impl Allocatedp {
+    pub fn default() -> Self {
+        #[cfg(unix)]
+        {
+            let allocated = thread::allocatedp::mib().unwrap();
+            let allocated = allocated.read().unwrap();
+            Self { allocated }
+        }
+        #[cfg(not(unix))]
+        Self {}
+    }
+
+    /// Return current thread heap usage
+    pub fn get(&self) -> u64 {
+        #[cfg(unix)]
+        {
+            self.allocated.get()
+        }
+        #[cfg(not(unix))]
+        0
+    }
+
+    /// Return the difference in thread heap usage since a previous `get()`
+    pub fn since(&self, previous: u64) -> i64 {
+        self.get() as i64 - previous as i64
+    }
+}