removes delayed crds inserts when upserting gossip table (#16806)

It is crucial that VersionedCrdsValue::insert_timestamp does not go
backward in time:
https://github.com/solana-labs/solana/blob/ec37a843a/core/src/crds.rs#L67-L79

Otherwise methods such as get_votes and get_epoch_slots_since will
break, which will break their downstream flow, including vote-listener
and optimistic confirmation:
https://github.com/solana-labs/solana/blob/ec37a843a/core/src/cluster_info.rs#L1197-L1215
https://github.com/solana-labs/solana/blob/ec37a843a/core/src/cluster_info.rs#L1274-L1298

For that, Crds::new_versioned is intended to be called "atomically" with
Crds::insert_verioned (as the comment already says so):
https://github.com/solana-labs/solana/blob/ec37a843a/core/src/crds.rs#L126-L129

However, currently this is violated in the code. For example,
filter_pull_responses creates VersionedCrdsValues (with the current
timestamp), then acquires an exclusive lock on gossip, then
process_pull_responses writes those values to the crds table:
https://github.com/solana-labs/solana/blob/ec37a843a/core/src/cluster_info.rs#L2375-L2392

Depending on the workload and lock contention, the insert_timestamps may
well be in the past when these values finally are inserted into gossip.

To avoid such scenarios, this commit:
  * removes Crds::new_versioned and Crd::insert_versioned.
  * makes VersionedCrdsValue constructor private, only invoked in
    Crds::insert, so that insert_timestamp is populated right before
    insert.

This will improve insert_timestamp monotonicity as long as Crds::insert
is not called with a stalled timestamp. Following commits may further
improve this by calling timestamp() inside Crds::insert, and/or
switching to std::time::Instant which guarantees monotonicity.

core/benches/crds_shards.rs

@@ -3,30 +3,34 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
-use solana_core::contact_info::ContactInfo;
-use solana_core::crds::VersionedCrdsValue;
-use solana_core::crds_shards::CrdsShards;
-use solana_core::crds_value::{CrdsData, CrdsValue};
-use solana_sdk::pubkey;
+use solana_core::{
+    crds::{Crds, VersionedCrdsValue},
+    crds_shards::CrdsShards,
+    crds_value::CrdsValue,
+};
 use solana_sdk::timing::timestamp;
+use std::iter::repeat_with;
 use test::Bencher;
 
 const CRDS_SHARDS_BITS: u32 = 8;
 
-fn new_test_crds_value() -> VersionedCrdsValue {
-    let data = CrdsData::ContactInfo(ContactInfo::new_localhost(&pubkey::new_rand(), timestamp()));
-    VersionedCrdsValue::new(timestamp(), CrdsValue::new_unsigned(data))
+fn new_test_crds_value<R: Rng>(rng: &mut R) -> VersionedCrdsValue {
+    let value = CrdsValue::new_rand(rng, None);
+    let label = value.label();
+    let mut crds = Crds::default();
+    crds.insert(value, timestamp()).unwrap();
+    crds.remove(&label).unwrap()
 }
 
 fn bench_crds_shards_find(bencher: &mut Bencher, num_values: usize, mask_bits: u32) {
-    let values: Vec<VersionedCrdsValue> = std::iter::repeat_with(new_test_crds_value)
+    let mut rng = thread_rng();
+    let values: Vec<_> = repeat_with(|| new_test_crds_value(&mut rng))
         .take(num_values)
         .collect();
     let mut shards = CrdsShards::new(CRDS_SHARDS_BITS);
     for (index, value) in values.iter().enumerate() {
         assert!(shards.insert(index, value));
     }
-    let mut rng = thread_rng();
     bencher.iter(|| {
         let mask = rng.gen();
         let _hits = shards.find(mask, mask_bits).count();