debug logs when crds table trim failed (#18307) (#18309)

reports of this error being possibly spammy:
https://discord.com/channels/428295358100013066/689412830075551748/859441080054710293

The commit changes the log level to debug.
Additionally adding a new metric to understand the frequency of this error.

(cherry picked from commit 9d983a34a0)

Co-authored-by: behzad nouri <behzadnouri@gmail.com>

gossip/src/cluster_info.rs

@@ -1761,11 +1761,14 @@ impl ClusterInfo {
             .map(|k| k.id)
             .chain(std::iter::once(self.id))
             .collect();
+        self.stats.trim_crds_table.add_relaxed(1);
         let mut gossip = self.gossip.write().unwrap();
         match gossip.crds.trim(cap, &keep, stakes, timestamp()) {
             Err(err) => {
                 self.stats.trim_crds_table_failed.add_relaxed(1);
-                error!("crds table trim failed: {:?}", err);
+                // TODO: Stakes are comming from the root-bank. Debug why/when
+                // they are empty/zero.
+                debug!("crds table trim failed: {:?}", err);
             }
             Ok(num_purged) => {
                 self.stats

gossip/src/cluster_info_metrics.rs

@@ -115,6 +115,7 @@ pub(crate) struct GossipStats {
     pub(crate) skip_pull_response_shred_version: Counter,
     pub(crate) skip_pull_shred_version: Counter,
     pub(crate) skip_push_message_shred_version: Counter,
+    pub(crate) trim_crds_table: Counter,
     pub(crate) trim_crds_table_failed: Counter,
     pub(crate) trim_crds_table_purged_values_count: Counter,
     pub(crate) tvu_peers: Counter,
@@ -398,6 +399,7 @@ pub(crate) fn submit_gossip_stats(
             stats.require_stake_for_gossip_unknown_stakes.clear(),
             i64
         ),
+        ("trim_crds_table", stats.trim_crds_table.clear(), i64),
         (
             "trim_crds_table_failed",
             stats.trim_crds_table_failed.clear(),

gossip/src/crds.rs

@@ -533,7 +533,7 @@ impl Crds {
         stakes: &HashMap<Pubkey, u64>,
         now: u64,
     ) -> Result</*num purged:*/ usize, CrdsError> {
-        if stakes.is_empty() {
+        if stakes.values().all(|&stake| stake == 0) {
             return Err(CrdsError::UnknownStakes);
         }
         let mut keys: Vec<_> = self