Gossip PullRequests tend to return a lot of duplicates. (#10326)

* filter messages that are likely to be pushed from the response

* tests

* tests

* wait to start filtering responses, and push stats to influx

* wait to start filtering responses, and push stats to influx

* reduce the timers to match the publish self timeout

* fmt

* fmt

core/src/cluster_info.rs

@@ -251,6 +251,7 @@ pub struct ClusterInfo {
     my_contact_info: RwLock<ContactInfo>,
     id: Pubkey,
     stats: GossipStats,
+    start_time: u64,
 }
 
 #[derive(Default, Clone)]
@@ -391,6 +392,7 @@ impl ClusterInfo {
         let id = contact_info.id;
         let me = Self {
             gossip: RwLock::new(CrdsGossip::default()),
+            start_time: timestamp(),
             keypair,
             entrypoint: RwLock::new(None),
             outbound_budget: RwLock::new(DataBudget {
@@ -425,6 +427,7 @@ impl ClusterInfo {
             my_contact_info: RwLock::new(my_contact_info),
             id: *new_id,
             stats: GossipStats::default(),
+            start_time: self.start_time,
         }
     }
 
@@ -1679,9 +1682,18 @@ impl ClusterInfo {
         let now = timestamp();
         let self_id = me.id();
 
+        //skip messages that are likely to be pushed
+        let min_filter_time = me.start_time + 10 * CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+        let push_timer = if min_filter_time < now {
+            // reason for / 3 is to allow push_self which has a /2 timeout to propagate
+            // first through push before responding with those values.
+            Some(now - CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS / 3)
+        } else {
+            None
+        };
         let pull_responses = me
             .time_gossip_read_lock("generate_pull_responses", &me.stats.generate_pull_responses)
-            .generate_pull_responses(&caller_and_filters);
+            .generate_pull_responses(&caller_and_filters, push_timer);
 
         me.time_gossip_write_lock("process_pull_reqs", &me.stats.process_pull_requests)
             .process_pull_requests(caller_and_filters, now);
@@ -2060,6 +2072,16 @@ impl ClusterInfo {
                     self.stats.generate_pull_responses.clear(),
                     i64
                 ),
+                (
+                    "process_pull_response_fail",
+                    self.stats.process_pull_response_fail.clear(),
+                    i64
+                ),
+                (
+                    "process_pull_response_success",
+                    self.stats.process_pull_response_success.clear(),
+                    i64
+                ),
                 ("process_prune", self.stats.process_prune.clear(), i64),
                 (
                     "process_push_message",