Derive Pod/Zeroable for Pubkey (backport #20493 ) (#20496 )

* Derive Pod/Zeroable for Pubkey (cherry picked from commit f966859829) # Conflicts: # Cargo.lock # programs/bpf/Cargo.lock # sdk/program/Cargo.toml * rebase Co-authored-by: Michael Vines <mvines@gmail.com>
fixes backports code changes (#20482 )
2021-10-07 07:59:17 +00:00 · 2021-10-07 03:44:30 +00:00 · 2021-10-06 16:51:31 -06:00 · 2021-10-06 16:25:24 -06:00 · 2021-10-06 15:11:41 -05:00
9 changed files with 82 additions and 25 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -468,6 +468,26 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b0017894339f586ccb943b01b9555de56770c11cda818e7e3d8bd93f4ed7f46e"

+[[package]]
+name = "bytemuck"
+version = "1.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e215f8c2f9f79cb53c8335e687ffd07d5bfcb6fe5fc80723762d0be46e7cc54"
+dependencies = [
+ "proc-macro2 1.0.24",
+ "quote 1.0.9",
+ "syn 1.0.60",
+]
+
 [[package]]
 name = "byteorder"
 version = "1.3.4"
@@ -5101,6 +5121,7 @@ dependencies = [
 "borsh-derive",
 "bs58",
 "bv",
+ "bytemuck",
 "curve25519-dalek 2.1.0",
 "hex",
 "itertools 0.9.0",
--- a/core/benches/cluster_info.rs
+++ b/core/benches/cluster_info.rs
@@ -75,9 +75,9 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
            &cluster_nodes_cache,
            &last_datapoint,
            &mut TransmitShredsStats::default(),
-            &SocketAddrSpace::Unspecified,
            &cluster_info,
            &bank_forks,
+            &SocketAddrSpace::Unspecified,
        )
        .unwrap();
    });
--- a/core/src/broadcast_stage.rs
+++ b/core/src/broadcast_stage.rs
@@ -22,8 +22,10 @@ use {
    solana_metrics::{inc_new_counter_error, inc_new_counter_info},
    solana_poh::poh_recorder::WorkingBankEntry,
    solana_runtime::{bank::Bank, bank_forks::BankForks},
-    solana_sdk::timing::{timestamp, AtomicInterval},
-    solana_sdk::{clock::Slot, pubkey::Pubkey},
+    solana_sdk::{
+        timing::{timestamp, AtomicInterval},
+        {clock::Slot, pubkey::Pubkey},
+    },
    solana_streamer::{
        sendmmsg::{batch_send, SendPktsError},
        socket::SocketAddrSpace,
@@ -31,9 +33,11 @@ use {
    std::{
        collections::HashMap,
        net::UdpSocket,
-        sync::atomic::{AtomicBool, Ordering},
-        sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
-        sync::{Arc, Mutex, RwLock},
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
+            Arc, Mutex, RwLock,
+        },
        thread::{self, Builder, JoinHandle},
        time::{Duration, Instant},
    },
@@ -399,9 +403,9 @@ pub fn broadcast_shreds(
    cluster_nodes_cache: &ClusterNodesCache<BroadcastStage>,
    last_datapoint_submit: &Arc<AtomicInterval>,
    transmit_stats: &mut TransmitShredsStats,
-    socket_addr_space: &SocketAddrSpace,
    cluster_info: &ClusterInfo,
    bank_forks: &Arc<RwLock<BankForks>>,
+    socket_addr_space: &SocketAddrSpace,
 ) -> Result<()> {
    let mut result = Ok(());
    let mut shred_select = Measure::start("shred_select");
--- a/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs
+++ b/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs
@@ -145,9 +145,9 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
            &self.cluster_nodes_cache,
            &Arc::new(AtomicInterval::default()),
            &mut TransmitShredsStats::default(),
-            cluster_info.socket_addr_space(),
            cluster_info,
            bank_forks,
+            cluster_info.socket_addr_space(),
        )
    }
    fn record(
--- a/core/src/broadcast_stage/standard_broadcast_run.rs
+++ b/core/src/broadcast_stage/standard_broadcast_run.rs
@@ -362,9 +362,9 @@ impl StandardBroadcastRun {
            &self.cluster_nodes_cache,
            &self.last_datapoint_submit,
            &mut transmit_stats,
-            cluster_info.socket_addr_space(),
            cluster_info,
            bank_forks,
+            cluster_info.socket_addr_space(),
        )?;
        transmit_time.stop();

--- a/programs/bpf/Cargo.lock
+++ b/programs/bpf/Cargo.lock
@@ -294,6 +294,26 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"

+[[package]]
+name = "bytemuck"
+version = "1.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e215f8c2f9f79cb53c8335e687ffd07d5bfcb6fe5fc80723762d0be46e7cc54"
+dependencies = [
+ "proc-macro2 1.0.24",
+ "quote 1.0.6",
+ "syn 1.0.67",
+]
+
 [[package]]
 name = "byteorder"
 version = "0.5.3"
@@ -3283,6 +3303,7 @@ dependencies = [
 "borsh-derive",
 "bs58",
 "bv",
+ "bytemuck",
 "curve25519-dalek 2.1.0",
 "hex",
 "itertools 0.9.0",
--- a/rpc/src/rpc_subscriptions.rs
+++ b/rpc/src/rpc_subscriptions.rs
@@ -213,6 +213,12 @@ impl RecentItems {
                .checked_sub(item.len())
                .expect("total bytes underflow");
        }
+
+        datapoint_info!(
+            "rpc_subscriptions_recent_items",
+            ("num", self.queue.len(), i64),
+            ("total_bytes", self.total_bytes, i64),
+        );
    }
 }

--- a/sdk/program/Cargo.toml
+++ b/sdk/program/Cargo.toml
@@ -15,6 +15,7 @@ bincode = "1.3.1"
 borsh = "0.9.0"
 borsh-derive = "0.9.0"
 bs58 = "0.3.1"
+bytemuck = { version = "1.7.2", features = ["derive"] }
 bv = { version = "0.11.1", features = ["serde"] }
 hex = "0.4.2"
 itertools =  "0.9.0"
--- a/sdk/program/src/pubkey.rs
+++ b/sdk/program/src/pubkey.rs
@@ -1,14 +1,16 @@
 #![allow(clippy::integer_arithmetic)]
-use crate::{decode_error::DecodeError, hash::hashv};
-
-use borsh::{BorshDeserialize, BorshSchema, BorshSerialize};
-use num_derive::{FromPrimitive, ToPrimitive};
-use std::{
+use {
+    crate::{decode_error::DecodeError, hash::hashv},
+    borsh::{BorshDeserialize, BorshSchema, BorshSerialize},
+    bytemuck::{Pod, Zeroable},
+    num_derive::{FromPrimitive, ToPrimitive},
+    std::{
        convert::{Infallible, TryFrom},
        fmt, mem,
        str::FromStr,
+    },
+    thiserror::Error,
 };
-use thiserror::Error;

 /// Number of bytes in a pubkey
 pub const PUBKEY_BYTES: usize = 32;
@@ -48,20 +50,22 @@ impl From<u64> for PubkeyError {

 #[repr(transparent)]
 #[derive(
-    Serialize,
-    Deserialize,
-    BorshSerialize,
+    AbiExample,
    BorshDeserialize,
    BorshSchema,
+    BorshSerialize,
    Clone,
    Copy,
    Default,
+    Deserialize,
    Eq,
-    PartialEq,
-    Ord,
-    PartialOrd,
    Hash,
-    AbiExample,
+    Ord,
+    PartialEq,
+    PartialOrd,
+    Pod,
+    Serialize,
+    Zeroable,
 )]
 pub struct Pubkey([u8; 32]);
Author	SHA1	Message	Date
mergify[bot]	4892eb4e1a	Derive Pod/Zeroable for Pubkey (backport #20493 ) (#20496 ) * Derive Pod/Zeroable for Pubkey (cherry picked from commit `f966859829`) # Conflicts: # Cargo.lock # programs/bpf/Cargo.lock # sdk/program/Cargo.toml * rebase Co-authored-by: Michael Vines <mvines@gmail.com>	2021-10-07 07:59:17 +00:00
behzad nouri	2d930052dc	fixes backports code changes (#20482 )	2021-10-07 03:44:30 +00:00
mergify[bot]	70c9ca061f	Add RecentItems metrics (#20484 ) (#20487 ) (cherry picked from commit `e13ed8a627`) Co-authored-by: Tyera Eulberg <teulberg@gmail.com>	2021-10-06 16:51:31 -06:00
Trent Nelson	e4aecd9320	Revert "Cost model 1.7 (#20188 )" This reverts commit `1dd6dc3709`.	2021-10-06 16:25:24 -06:00
Tao Zhu	1dd6dc3709	Cost model 1.7 (#20188 ) * Cost Model to limit transactions which are not parallelizeable (#16694) * * Add following to banking_stage: 1. CostModel as immutable ref shared between threads, to provide estimated cost for transactions. 2. CostTracker which is shared between threads, tracks transaction costs for each block. * replace hard coded program ID with id() calls * Add Account Access Cost as part of TransactionCost. Account Access cost are weighted differently between read and write, signed and non-signed. * Establish instruction_execution_cost_table, add function to update or insert instruction cost, unit tested. It is read-only for now; it allows Replay to insert realtime instruction execution costs to the table. * add test for cost_tracker atomically try_add operation, serves as safety guard for future changes * check cost against local copy of cost_tracker, return transactions that would exceed limit as unprocessed transaction to be buffered; only apply bank processed transactions cost to tracker; * bencher to new banking_stage with max cost limit to allow cost model being hit consistently during bench iterations * replay stage feed back program cost (#17731) * replay stage feeds back realtime per-program execution cost to cost model; * program cost execution table is initialized into empty table, no longer populated with hardcoded numbers; * changed cost unit to microsecond, using value collected from mainnet; * add ExecuteCostTable with fixed capacity for security concern, when its limit is reached, programs with old age AND less occurrence will be pushed out to make room for new programs. * investigate system performance test degradation (#17919) * Add stats and counter around cost model ops, mainly: - calculate transaction cost - check transaction can fit in a block - update block cost tracker after transactions are added to block - replay_stage to update/insert execution cost to table * Change mutex on cost_tracker to RwLock * removed cloning cost_tracker for local use, as the metrics show clone is very expensive. * acquire and hold locks for block of TXs, instead of acquire and release per transaction; * remove redundant would_fit check from cost_tracker update execution path * refactor cost checking with less frequent lock acquiring * avoid many Transaction_cost heap allocation when calculate cost, which is in the hot path - executed per transaction. * create hashmap with new_capacity to reduce runtime heap realloc. * code review changes: categorize stats, replace explicit drop calls, concisely initiate to default * address potential deadlock by acquiring locks one at time * Persist cost table to blockstore (#18123) * Add `ProgramCosts` Column Family to blockstore, implement LedgerColumn; add `delete_cf` to Rocks * Add ProgramCosts to compaction excluding list alone side with TransactionStatusIndex in one place: `excludes_from_compaction()` * Write cost table to blockstore after `replay_stage` replayed active banks; add stats to measure persist time * Deletes program from `ProgramCosts` in blockstore when they are removed from cost_table in memory * Only try to persist to blockstore when cost_table is changed. * Restore cost table during validator startup * Offload `cost_model` related operations from replay main thread to dedicated service thread, add channel to send execute_timings between these threads; * Move `cost_update_service` to its own module; replay_stage is now decoupled from cost_model. * log warning when channel send fails (#18391) * Aggregate cost_model into cost_tracker (#18374) * * aggregate cost_model into cost_tracker, decouple it from banking_stage to prevent accidental deadlock. * Simplified code, removed unused functions * review fixes * update ledger tool to restore cost table from blockstore (#18489) * update ledger tool to restore cost model from blockstore when compute-slot-cost * Move initialize_cost_table into cost_model, so the function can be tested and shared between validator and ledger-tool * refactor and simplify a test * manually fix merge conflicts * Per-program id timings (#17554) * more manual fixing * solve a merge conflict * featurize cost model * more merge fix * cost model uses compute_unit to replace microsecond as cost unit (#18934) * Reject blocks for costs above the max block cost (#18994) * Update block max cost limit to fix performance regession (#19276) * replace function with const var for better readability (#19285) * Add few more metrics data points (#19624) * periodically report sigverify_stage stats (#19674) * manual merge * cost model nits (#18528) * Accumulate consumed units (#18714) * tx wide compute budget (#18631) * more manual merge * ignore zerorize drop security * - update const cost values with data collected by #19627 - update cost calculation to closely proposed fee schedule #16984 * add transaction cost histogram metrics (#20350) * rebase to 1.7.15 * add tx count and thread id to stats (#20451) each stat reports and resets when slot changes * remove cost_model feature_set * ignore vote transactions from cost model Co-authored-by: sakridge <sakridge@gmail.com> Co-authored-by: Jeff Biseda <jbiseda@gmail.com> Co-authored-by: Jack May <jack@solana.com>	2021-10-06 15:11:41 -05:00