diff --git a/Cargo.lock b/Cargo.lock index 875f701638..027ef86351 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3268,12 +3268,10 @@ dependencies = [ "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "bs58 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", "core_affinity 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "dir-diff 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3322,7 +3320,6 @@ dependencies = [ "solana-vote-signer 0.20.0", "symlink 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3534,9 +3531,12 @@ version = "0.20.0" dependencies = [ "bincode 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "dir-diff 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "dlopen 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "dlopen_derive 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3549,6 +3549,7 @@ dependencies = [ "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "solana-budget-api 0.20.0", "solana-logger 0.20.0", + "solana-measure 0.20.0", "solana-merkle-tree 0.20.0", "solana-metrics 0.20.0", "solana-rayon-threadlimit 0.20.0", @@ -3558,6 +3559,8 @@ dependencies = [ "solana-stake-api 0.20.0", "solana-vote-api 0.20.0", "sys-info 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3571,7 +3574,6 @@ dependencies = [ "serde_derive 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)", - "solana-core 0.20.0", "solana-ledger 0.20.0", "solana-logger 0.20.0", "solana-runtime 0.20.0", diff --git a/banking_bench/src/main.rs b/banking_bench/src/main.rs index aa5a483e2a..4ddd0683f5 100644 --- a/banking_bench/src/main.rs +++ b/banking_bench/src/main.rs @@ -6,7 +6,6 @@ use crossbeam_channel::unbounded; use log::*; use rand::{thread_rng, Rng}; use rayon::prelude::*; -use solana_core::bank_forks::BankForks; use solana_core::banking_stage::{create_test_recorder, BankingStage}; use solana_core::cluster_info::ClusterInfo; use solana_core::cluster_info::Node; @@ -15,6 +14,7 @@ use solana_core::packet::to_packets_chunked; use solana_core::poh_recorder::PohRecorder; use solana_core::poh_recorder::WorkingBankEntry; use solana_core::service::Service; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; use solana_measure::measure::Measure; use solana_runtime::bank::Bank; diff --git a/core/Cargo.toml b/core/Cargo.toml index 62458d9052..ec52c9cab6 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -21,12 +21,10 @@ move = [] bincode = "1.2.0" bs58 = "0.3.0" byteorder = "1.3.2" -bzip2 = "0.3.3" chrono = { version = "0.4.9", features = ["serde"] } core_affinity = "0.5.9" crc = { version = "1.8.1", optional = true } crossbeam-channel = "0.3" -dir-diff = "0.3.2" fs_extra = "1.1.0" indexmap = "1.1" itertools = "0.8.0" @@ -68,7 +66,6 @@ solana-vote-api = { path = "../programs/vote_api", version = "0.20.0" } solana-vote-signer = { path = "../vote-signer", version = "0.20.0" } symlink = "0.1.0" sys-info = "0.5.8" -tar = "0.4.26" tempfile = "3.1.0" tokio = "0.1" tokio-codec = "0.1" diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index 15a928f771..149749558d 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -9,7 +9,6 @@ use log::*; use rand::{thread_rng, Rng}; use rayon::prelude::*; use solana_core::banking_stage::{create_test_recorder, BankingStage}; -use solana_core::blocktree_processor::process_entries; use solana_core::cluster_info::ClusterInfo; use solana_core::cluster_info::Node; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; @@ -18,6 +17,7 @@ use solana_core::poh_recorder::WorkingBankEntry; use solana_core::service::Service; use solana_core::test_tx::test_tx; use solana_ledger::blocktree::{get_tmp_ledger_path, Blocktree}; +use solana_ledger::blocktree_processor::process_entries; use solana_ledger::entry::{next_hash, Entry}; use solana_runtime::bank::Bank; use solana_sdk::genesis_block::GenesisBlock; diff --git a/core/benches/retransmit_stage.rs b/core/benches/retransmit_stage.rs index c4f68bda06..116c982a77 100644 --- a/core/benches/retransmit_stage.rs +++ b/core/benches/retransmit_stage.rs @@ -4,13 +4,13 @@ extern crate solana_core; extern crate test; use log::*; -use solana_core::bank_forks::BankForks; use solana_core::cluster_info::{ClusterInfo, Node}; use solana_core::contact_info::ContactInfo; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use solana_core::packet::to_packets_chunked; use solana_core::retransmit_stage::retransmitter; use solana_core::test_tx::test_tx; +use solana_ledger::bank_forks::BankForks; use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_measure::measure::Measure; use solana_runtime::bank::Bank; diff --git a/core/src/cluster_info.rs b/core/src/cluster_info.rs index e9eb38de0c..56be0e80f3 100644 --- a/core/src/cluster_info.rs +++ b/core/src/cluster_info.rs @@ -12,7 +12,6 @@ //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes. //! //! Bank needs to provide an interface for us to query the stake weight -use crate::bank_forks::BankForks; use crate::contact_info::ContactInfo; use crate::crds_gossip::CrdsGossip; use crate::crds_gossip_error::CrdsGossipError; @@ -30,6 +29,7 @@ use itertools::Itertools; use rand::SeedableRng; use rand::{thread_rng, Rng}; use rand_chacha::ChaChaRng; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::Blocktree; use solana_ledger::staking_utils; use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error}; @@ -1779,7 +1779,6 @@ fn report_time_spent(label: &str, time: &Duration, extra: &str) { #[cfg(test)] mod tests { use super::*; - use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks; use crate::crds_value::CrdsValueLabel; use crate::repair_service::RepairType; use crate::result::Error; @@ -1788,6 +1787,7 @@ mod tests { use solana_ledger::blocktree::get_tmp_ledger_path; use solana_ledger::blocktree::make_many_slot_entries; use solana_ledger::blocktree::Blocktree; + use solana_ledger::blocktree_processor::fill_blocktree_slot_with_ticks; use solana_ledger::shred::{ max_ticks_per_n_shreds, CodingShredHeader, DataShredHeader, Shred, ShredCommonHeader, }; diff --git a/core/src/cluster_info_repair_listener.rs b/core/src/cluster_info_repair_listener.rs index 75d55092bf..60507b2764 100644 --- a/core/src/cluster_info_repair_listener.rs +++ b/core/src/cluster_info_repair_listener.rs @@ -1,13 +1,13 @@ use crate::cluster_info::ClusterInfo; use crate::crds_value::EpochSlots; use crate::result::Result; -use crate::rooted_slot_iterator::RootedSlotIterator; use crate::service::Service; use byteorder::{ByteOrder, LittleEndian}; use rand::seq::SliceRandom; use rand::SeedableRng; use rand_chacha::ChaChaRng; use solana_ledger::blocktree::Blocktree; +use solana_ledger::rooted_slot_iterator::RootedSlotIterator; use solana_metrics::datapoint; use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey}; use std::{ diff --git a/core/src/consensus.rs b/core/src/consensus.rs index aaa5fa8251..ff75725d47 100644 --- a/core/src/consensus.rs +++ b/core/src/consensus.rs @@ -1,4 +1,4 @@ -use crate::bank_forks::BankForks; +use solana_ledger::bank_forks::BankForks; use solana_metrics::datapoint_debug; use solana_runtime::bank::Bank; use solana_sdk::account::Account; diff --git a/core/src/gossip_service.rs b/core/src/gossip_service.rs index c82f8f956f..21685385c1 100644 --- a/core/src/gossip_service.rs +++ b/core/src/gossip_service.rs @@ -1,12 +1,12 @@ //! The `gossip_service` module implements the network control plane. -use crate::bank_forks::BankForks; use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE}; use crate::contact_info::ContactInfo; use crate::service::Service; use crate::streamer; use rand::{thread_rng, Rng}; use solana_client::thin_client::{create_client, ThinClient}; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::Blocktree; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; diff --git a/core/src/lib.rs b/core/src/lib.rs index ed21f93b07..232fd43c7c 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -5,7 +5,6 @@ //! command-line tools to spin up validators and a Rust library //! -pub mod bank_forks; pub mod banking_stage; pub mod broadcast_stage; pub mod chacha; @@ -18,7 +17,6 @@ pub mod shred_fetch_stage; pub mod contact_info; pub mod blockstream; pub mod blockstream_service; -pub mod blocktree_processor; pub mod cluster_info; pub mod cluster_info_repair_listener; pub mod consensus; @@ -44,7 +42,6 @@ pub mod replay_stage; pub mod replicator; pub mod result; pub mod retransmit_stage; -pub mod rooted_slot_iterator; pub mod rpc; pub mod rpc_pubsub; pub mod rpc_pubsub_service; @@ -54,9 +51,7 @@ pub mod sendmmsg; pub mod service; pub mod sigverify; pub mod sigverify_stage; -pub mod snapshot_package; pub mod snapshot_packager_service; -pub mod snapshot_utils; pub mod storage_stage; pub mod streamer; pub mod test_tx; @@ -70,8 +65,6 @@ pub mod window_service; #[macro_use] extern crate solana_budget_program; -extern crate solana_storage_program; - #[cfg(test)] #[macro_use] extern crate hex_literal; @@ -95,11 +88,3 @@ extern crate matches; #[macro_use] extern crate solana_ledger; - -extern crate bzip2; -extern crate crossbeam_channel; -extern crate dir_diff; -extern crate fs_extra; -extern crate symlink; -extern crate tar; -extern crate tempfile; diff --git a/core/src/repair_service.rs b/core/src/repair_service.rs index 8f173dffe1..b3603b8675 100644 --- a/core/src/repair_service.rs +++ b/core/src/repair_service.rs @@ -1,9 +1,10 @@ //! The `repair_service` module implements the tools necessary to generate a thread which //! regularly finds missing blobs in the ledger and sends repair requests for those blobs use crate::{ - bank_forks::BankForks, cluster_info::ClusterInfo, - cluster_info_repair_listener::ClusterInfoRepairListener, result::Result, service::Service, + cluster_info::ClusterInfo, cluster_info_repair_listener::ClusterInfoRepairListener, + result::Result, service::Service, }; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver, SlotMeta}; use solana_sdk::{epoch_schedule::EpochSchedule, pubkey::Pubkey}; use std::{ diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index bef262ffc3..6d788be9da 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1,7 +1,5 @@ //! The `replay_stage` replays transactions broadcast by the leader. -use crate::bank_forks::BankForks; -use crate::blocktree_processor; use crate::cluster_info::ClusterInfo; use crate::confidence::{ AggregateConfidenceService, ConfidenceAggregationData, ForkConfidenceCache, @@ -12,10 +10,12 @@ use crate::poh_recorder::PohRecorder; use crate::result::{Error, Result}; use crate::rpc_subscriptions::RpcSubscriptions; use crate::service::Service; -use crate::snapshot_package::SnapshotPackageSender; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::{Blocktree, BlocktreeError}; +use solana_ledger::blocktree_processor; use solana_ledger::entry::{Entry, EntrySlice}; use solana_ledger::leader_schedule_cache::LeaderScheduleCache; +use solana_ledger::snapshot_package::SnapshotPackageSender; use solana_metrics::{datapoint_warn, inc_new_counter_info}; use solana_runtime::bank::Bank; use solana_sdk::hash::Hash; diff --git a/core/src/result.rs b/core/src/result.rs index f6aa73572e..ebada594a7 100644 --- a/core/src/result.rs +++ b/core/src/result.rs @@ -3,12 +3,9 @@ use crate::cluster_info; use crate::packet; use crate::poh_recorder; -use crate::snapshot_utils; -use bincode; -use serde_json; use solana_ledger::blocktree; +use solana_ledger::snapshot_utils; use solana_sdk::transaction; -use std; use std::any::Any; #[derive(Debug)] diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index 59c80d959a..1c7cc12e2b 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -1,7 +1,6 @@ //! The `retransmit_stage` retransmits blobs between validators use crate::{ - bank_forks::BankForks, cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT}, repair_service::RepairStrategy, result::{Error, Result}, @@ -12,6 +11,7 @@ use crate::{ use rand::SeedableRng; use rand_chacha::ChaChaRng; use solana_ledger::{ + bank_forks::BankForks, blocktree::{Blocktree, CompletedSlotsReceiver}, leader_schedule_cache::LeaderScheduleCache, staking_utils, @@ -262,11 +262,11 @@ impl Service for RetransmitStage { #[cfg(test)] mod tests { use super::*; - use crate::blocktree_processor::{process_blocktree, ProcessOptions}; use crate::contact_info::ContactInfo; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use crate::packet::{Meta, Packet, Packets}; use solana_ledger::blocktree::create_new_tmp_ledger; + use solana_ledger::blocktree_processor::{process_blocktree, ProcessOptions}; use solana_netutil::find_available_port_in_range; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 7388a1f184..b0a44d82c7 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -1,7 +1,6 @@ //! The `rpc` module implements the Solana RPC interface. use crate::{ - bank_forks::BankForks, cluster_info::ClusterInfo, confidence::{BankConfidence, ForkConfidenceCache}, contact_info::ContactInfo, @@ -15,6 +14,7 @@ use jsonrpc_core::{Error, Metadata, Result}; use jsonrpc_derive::rpc; use solana_client::rpc_request::{RpcEpochInfo, RpcVoteAccountInfo, RpcVoteAccountStatus}; use solana_drone::drone::request_airdrop_transaction; +use solana_ledger::bank_forks::BankForks; use solana_runtime::bank::Bank; use solana_sdk::{ account::Account, diff --git a/core/src/rpc_pubsub.rs b/core/src/rpc_pubsub.rs index 42bbf0b484..1e154e1364 100644 --- a/core/src/rpc_pubsub.rs +++ b/core/src/rpc_pubsub.rs @@ -238,13 +238,13 @@ impl RpcSolPubSub for RpcSolPubSubImpl { #[cfg(test)] mod tests { use super::*; - use crate::bank_forks::BankForks; use crate::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use jsonrpc_core::futures::sync::mpsc; use jsonrpc_core::Response; use jsonrpc_pubsub::{PubSubHandler, Session}; use solana_budget_api; use solana_budget_api::budget_instruction; + use solana_ledger::bank_forks::BankForks; use solana_runtime::bank::Bank; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, KeypairUtil}; diff --git a/core/src/rpc_service.rs b/core/src/rpc_service.rs index 69bc6e4ab9..092e6c1090 100644 --- a/core/src/rpc_service.rs +++ b/core/src/rpc_service.rs @@ -1,14 +1,15 @@ //! The `rpc_service` module implements the Solana JSON RPC service. use crate::{ - bank_forks::BankForks, cluster_info::ClusterInfo, confidence::ForkConfidenceCache, rpc::*, - service::Service, storage_stage::StorageState, validator::ValidatorExit, + cluster_info::ClusterInfo, confidence::ForkConfidenceCache, rpc::*, service::Service, + storage_stage::StorageState, validator::ValidatorExit, }; use jsonrpc_core::MetaIoHandler; use jsonrpc_http_server::{ hyper, AccessControlAllowOrigin, CloseHandle, DomainsValidation, RequestMiddleware, RequestMiddlewareAction, ServerBuilder, }; +use solana_ledger::bank_forks::BankForks; use solana_sdk::hash::Hash; use std::{ net::SocketAddr, diff --git a/core/src/rpc_subscriptions.rs b/core/src/rpc_subscriptions.rs index cb7aefa274..d49eeb4fcd 100644 --- a/core/src/rpc_subscriptions.rs +++ b/core/src/rpc_subscriptions.rs @@ -1,11 +1,11 @@ //! The `pubsub` module implements a threaded subscription service on client RPC request -use crate::bank_forks::BankForks; use core::hash::Hash; use jsonrpc_core::futures::Future; use jsonrpc_pubsub::typed::Sink; use jsonrpc_pubsub::SubscriptionId; use serde::Serialize; +use solana_ledger::bank_forks::BankForks; use solana_runtime::bank::Bank; use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/snapshot_packager_service.rs b/core/src/snapshot_packager_service.rs index c3eb26048b..1462ded332 100644 --- a/core/src/snapshot_packager_service.rs +++ b/core/src/snapshot_packager_service.rs @@ -1,8 +1,8 @@ use crate::result::{Error, Result}; use crate::service::Service; -use crate::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver}; -use crate::snapshot_utils::{self, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR}; use bincode::serialize_into; +use solana_ledger::snapshot_package::{SnapshotPackage, SnapshotPackageReceiver}; +use solana_ledger::snapshot_utils::{self, TAR_ACCOUNTS_DIR, TAR_SNAPSHOTS_DIR}; use solana_measure::measure::Measure; use solana_metrics::datapoint_info; use solana_runtime::status_cache::SlotDelta; @@ -187,7 +187,7 @@ impl Service for SnapshotPackagerService { #[cfg(test)] mod tests { use super::*; - use crate::snapshot_utils; + use solana_ledger::snapshot_utils; use solana_runtime::accounts_db::AccountStorageEntry; use std::fs::OpenOptions; use std::io::Write; diff --git a/core/src/storage_stage.rs b/core/src/storage_stage.rs index 26721777f2..8a6e3faa9a 100644 --- a/core/src/storage_stage.rs +++ b/core/src/storage_stage.rs @@ -2,7 +2,6 @@ // for storage mining. Replicators submit storage proofs, validator then bundles them // to submit its proof for mining to be rewarded. -use crate::bank_forks::BankForks; use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys; use crate::cluster_info::ClusterInfo; use crate::contact_info::ContactInfo; @@ -10,6 +9,7 @@ use crate::result::{Error, Result}; use crate::service::Service; use rand::{Rng, SeedableRng}; use rand_chacha::ChaChaRng; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::Blocktree; use solana_runtime::bank::Bank; use solana_runtime::storage_utils::replicator_accounts; diff --git a/core/src/tvu.rs b/core/src/tvu.rs index d0642d17ee..6b194bcee1 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -12,7 +12,6 @@ //! 4. StorageStage //! - Generating the keys used to encrypt the ledger and sample it for storage mining. -use crate::bank_forks::BankForks; use crate::blockstream_service::BlockstreamService; use crate::cluster_info::ClusterInfo; use crate::confidence::ForkConfidenceCache; @@ -25,6 +24,7 @@ use crate::service::Service; use crate::shred_fetch_stage::ShredFetchStage; use crate::snapshot_packager_service::SnapshotPackagerService; use crate::storage_stage::{StorageStage, StorageState}; +use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver}; use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/validator.rs b/core/src/validator.rs index 68ca0431dc..b1406a67bb 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1,7 +1,5 @@ //! The `validator` module hosts all the validator microservices. -use crate::bank_forks::{BankForks, SnapshotConfig}; -use crate::blocktree_processor::{self, BankForksInfo}; use crate::broadcast_stage::BroadcastStageType; use crate::cluster_info::{ClusterInfo, Node}; use crate::confidence::ForkConfidenceCache; @@ -15,12 +13,14 @@ use crate::rpc_service::JsonRpcService; use crate::rpc_subscriptions::RpcSubscriptions; use crate::service::Service; use crate::sigverify; -use crate::snapshot_utils; use crate::storage_stage::StorageState; use crate::tpu::Tpu; use crate::tvu::{Sockets, Tvu}; +use solana_ledger::bank_forks::{BankForks, SnapshotConfig}; use solana_ledger::blocktree::{Blocktree, CompletedSlotsReceiver}; +use solana_ledger::blocktree_processor::{self, BankForksInfo}; use solana_ledger::leader_schedule_cache::LeaderScheduleCache; +use solana_ledger::snapshot_utils; use solana_metrics::datapoint_info; use solana_sdk::clock::{Slot, DEFAULT_SLOTS_PER_TURN}; use solana_sdk::genesis_block::GenesisBlock; diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 52d628092f..ccc27535c1 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -5,11 +5,11 @@ mod tests { use bincode::serialize_into; use fs_extra::dir::CopyOptions; use itertools::Itertools; - use solana_core::bank_forks::{BankForks, SnapshotConfig}; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use solana_core::service::Service; use solana_core::snapshot_packager_service::SnapshotPackagerService; - use solana_core::snapshot_utils; + use solana_ledger::bank_forks::{BankForks, SnapshotConfig}; + use solana_ledger::snapshot_utils; use solana_runtime::bank::Bank; use solana_runtime::status_cache::SlotDelta; use solana_runtime::status_cache::MAX_CACHE_ENTRIES; diff --git a/core/tests/storage_stage.rs b/core/tests/storage_stage.rs index cb0b22b1ea..e7d5c3faed 100644 --- a/core/tests/storage_stage.rs +++ b/core/tests/storage_stage.rs @@ -6,13 +6,13 @@ extern crate solana_ledger; #[cfg(test)] mod tests { use log::*; - use solana_core::bank_forks::BankForks; - use solana_core::blocktree_processor; use solana_core::genesis_utils::{create_genesis_block, GenesisBlockInfo}; use solana_core::service::Service; use solana_core::storage_stage::{test_cluster_info, SLOTS_PER_TURN_TEST}; use solana_core::storage_stage::{StorageStage, StorageState}; + use solana_ledger::bank_forks::BankForks; use solana_ledger::blocktree::{create_new_tmp_ledger, Blocktree}; + use solana_ledger::blocktree_processor; use solana_ledger::entry; use solana_runtime::bank::Bank; use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT; diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 68317858c3..a4ba0d01d1 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -15,7 +15,6 @@ serde = "1.0.101" serde_derive = "1.0.101" serde_json = "1.0.41" serde_yaml = "0.8.11" -solana-core = { path = "../core", version = "0.20.0" } solana-ledger = { path = "../ledger", version = "0.20.0" } solana-logger = { path = "../logger", version = "0.20.0" } solana-runtime = { path = "../runtime", version = "0.20.0" } diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index ea9c3b6cc7..2d49b8523b 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1,7 +1,7 @@ use clap::{crate_description, crate_name, crate_version, value_t_or_exit, App, Arg, SubCommand}; -use solana_core::blocktree_processor::{process_blocktree, ProcessOptions}; -use solana_core::rooted_slot_iterator::RootedSlotIterator; use solana_ledger::blocktree::Blocktree; +use solana_ledger::blocktree_processor::{process_blocktree, ProcessOptions}; +use solana_ledger::rooted_slot_iterator::RootedSlotIterator; use solana_sdk::clock::Slot; use solana_sdk::genesis_block::GenesisBlock; use std::collections::BTreeMap; diff --git a/ledger-tool/tests/basic.rs b/ledger-tool/tests/basic.rs index 35242fbd68..0f5facc0e5 100644 --- a/ledger-tool/tests/basic.rs +++ b/ledger-tool/tests/basic.rs @@ -2,8 +2,8 @@ extern crate solana_ledger; use assert_cmd::prelude::*; -use solana_core::genesis_utils::create_genesis_block; use solana_ledger::blocktree::create_new_tmp_ledger; +use solana_ledger::genesis_utils::create_genesis_block; use std::process::Command; use std::process::Output; diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 4c6ee95250..110d95e710 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -11,9 +11,12 @@ edition = "2018" [dependencies] bincode = "1.2.0" byteorder = "1.3.2" +bzip2 = "0.3.3" chrono = { version = "0.4.9", features = ["serde"] } +dir-diff = "0.3.2" dlopen = "0.1.8" dlopen_derive = "0.1.4" +fs_extra = "1.1.0" itertools = "0.8.0" lazy_static = "1.4.0" log = { version = "0.4.8" } @@ -24,6 +27,7 @@ reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0 serde = "1.0.101" serde_derive = "1.0.101" solana-logger = { path = "../logger", version = "0.20.0" } +solana-measure = { path = "../measure", version = "0.20.0" } solana-merkle-tree = { path = "../merkle-tree", version = "0.20.0" } solana-metrics = { path = "../metrics", version = "0.20.0" } solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.20.0" } @@ -32,6 +36,8 @@ solana-sdk = { path = "../sdk", version = "0.20.0" } solana-stake-api = { path = "../programs/stake_api", version = "0.20.0" } solana-vote-api = { path = "../programs/vote_api", version = "0.20.0" } sys-info = "0.5.8" +tar = "0.4.26" +tempfile = "3.1.0" [dependencies.rocksdb] # Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts diff --git a/core/src/bank_forks.rs b/ledger/src/bank_forks.rs similarity index 94% rename from core/src/bank_forks.rs rename to ledger/src/bank_forks.rs index 66e9c240a9..83d4c44762 100644 --- a/core/src/bank_forks.rs +++ b/ledger/src/bank_forks.rs @@ -1,8 +1,8 @@ //! The `bank_forks` module implments BankForks a DAG of checkpointed Banks -use crate::result::Result; -use crate::snapshot_package::SnapshotPackageSender; -use crate::snapshot_utils; +use crate::snapshot_package::{SnapshotPackageSendError, SnapshotPackageSender}; +use crate::snapshot_utils::{self, SnapshotError}; +use log::*; use solana_measure::measure::Measure; use solana_metrics::inc_new_counter_info; use solana_runtime::bank::Bank; @@ -26,6 +26,25 @@ pub struct SnapshotConfig { pub snapshot_path: PathBuf, } +#[derive(Debug)] +pub enum BankForksError { + SnapshotError(SnapshotError), + SnapshotPackageSendError(SnapshotPackageSendError), +} +pub type Result = std::result::Result; + +impl std::convert::From for BankForksError { + fn from(e: SnapshotError) -> BankForksError { + BankForksError::SnapshotError(e) + } +} + +impl std::convert::From for BankForksError { + fn from(e: SnapshotPackageSendError) -> BankForksError { + BankForksError::SnapshotPackageSendError(e) + } +} + pub struct BankForks { pub banks: HashMap>, working_bank: Arc, diff --git a/core/src/blocktree_processor.rs b/ledger/src/blocktree_processor.rs similarity index 98% rename from core/src/blocktree_processor.rs rename to ledger/src/blocktree_processor.rs index bcd24fc38a..e56b4daa67 100644 --- a/core/src/blocktree_processor.rs +++ b/ledger/src/blocktree_processor.rs @@ -1,17 +1,19 @@ use crate::bank_forks::BankForks; +use crate::blocktree::{Blocktree, SlotMeta}; +use crate::entry::{create_ticks, Entry, EntrySlice}; +use crate::leader_schedule_cache::LeaderScheduleCache; +use log::*; use rand::seq::SliceRandom; use rand::thread_rng; use rayon::prelude::*; use rayon::ThreadPool; -use solana_ledger::blocktree::{Blocktree, SlotMeta}; -use solana_ledger::entry::{Entry, EntrySlice}; -use solana_ledger::leader_schedule_cache::LeaderScheduleCache; use solana_metrics::{datapoint, datapoint_error, inc_new_counter_debug}; use solana_runtime::bank::Bank; use solana_runtime::transaction_batch::TransactionBatch; use solana_sdk::clock::{Slot, MAX_RECENT_BLOCKHASHES}; use solana_sdk::genesis_block::GenesisBlock; use solana_sdk::hash::Hash; +use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::timing::duration_as_ms; use solana_sdk::transaction::Result; use std::result; @@ -447,15 +449,42 @@ fn process_pending_slots( Ok(fork_info) } +pub fn fill_blocktree_slot_with_ticks( + blocktree: &Blocktree, + ticks_per_slot: u64, + slot: u64, + parent_slot: u64, + last_entry_hash: Hash, +) -> Hash { + let entries = create_ticks(ticks_per_slot, last_entry_hash); + let last_entry_hash = entries.last().unwrap().hash; + + blocktree + .write_entries( + slot, + 0, + 0, + ticks_per_slot, + Some(parent_slot), + true, + &Arc::new(Keypair::new()), + entries, + ) + .unwrap(); + + last_entry_hash +} + #[cfg(test)] pub mod tests { use super::*; + use crate::blocktree::create_new_tmp_ledger; + use crate::entry::{create_ticks, next_entry, next_entry_mut, Entry}; use crate::genesis_utils::{ create_genesis_block, create_genesis_block_with_leader, GenesisBlockInfo, }; + use matches::assert_matches; use rand::{thread_rng, Rng}; - use solana_ledger::blocktree::create_new_tmp_ledger; - use solana_ledger::entry::{create_ticks, next_entry, next_entry_mut, Entry}; use solana_sdk::{ epoch_schedule::EpochSchedule, hash::Hash, @@ -467,32 +496,6 @@ pub mod tests { }; use std::sync::RwLock; - pub fn fill_blocktree_slot_with_ticks( - blocktree: &Blocktree, - ticks_per_slot: u64, - slot: u64, - parent_slot: u64, - last_entry_hash: Hash, - ) -> Hash { - let entries = create_ticks(ticks_per_slot, last_entry_hash); - let last_entry_hash = entries.last().unwrap().hash; - - blocktree - .write_entries( - slot, - 0, - 0, - ticks_per_slot, - Some(parent_slot), - true, - &Arc::new(Keypair::new()), - entries, - ) - .unwrap(); - - last_entry_hash - } - #[test] fn test_process_blocktree_with_incomplete_slot() { solana_logger::setup(); diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index e28e1605f7..92bb8f7a67 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -1,5 +1,7 @@ +pub mod bank_forks; #[macro_use] pub mod blocktree; +pub mod blocktree_processor; pub mod entry; pub mod erasure; pub mod genesis_utils; @@ -8,5 +10,11 @@ pub mod leader_schedule_cache; pub mod leader_schedule_utils; pub mod perf_libs; pub mod poh; +pub mod rooted_slot_iterator; pub mod shred; +pub mod snapshot_package; +pub mod snapshot_utils; pub mod staking_utils; + +#[macro_use] +extern crate solana_metrics; diff --git a/core/src/rooted_slot_iterator.rs b/ledger/src/rooted_slot_iterator.rs similarity index 97% rename from core/src/rooted_slot_iterator.rs rename to ledger/src/rooted_slot_iterator.rs index 3f76ed22db..c79f05fe20 100644 --- a/core/src/rooted_slot_iterator.rs +++ b/ledger/src/rooted_slot_iterator.rs @@ -1,4 +1,5 @@ -use solana_ledger::blocktree::*; +use crate::blocktree::*; +use log::*; pub struct RootedSlotIterator<'a> { next_slots: Vec, @@ -52,7 +53,7 @@ impl<'a> Iterator for RootedSlotIterator<'a> { #[cfg(test)] mod tests { use super::*; - use crate::blocktree_processor::tests::fill_blocktree_slot_with_ticks; + use crate::blocktree_processor::fill_blocktree_slot_with_ticks; use solana_sdk::hash::Hash; #[test] diff --git a/core/src/snapshot_package.rs b/ledger/src/snapshot_package.rs similarity index 89% rename from core/src/snapshot_package.rs rename to ledger/src/snapshot_package.rs index 39bc60d583..d4bbacdb17 100644 --- a/core/src/snapshot_package.rs +++ b/ledger/src/snapshot_package.rs @@ -2,12 +2,13 @@ use solana_runtime::accounts_db::AccountStorageEntry; use solana_runtime::status_cache::SlotDelta; use solana_sdk::transaction::Result as TransactionResult; use std::path::PathBuf; -use std::sync::mpsc::{Receiver, Sender}; +use std::sync::mpsc::{Receiver, SendError, Sender}; use std::sync::Arc; use tempfile::TempDir; pub type SnapshotPackageSender = Sender; pub type SnapshotPackageReceiver = Receiver; +pub type SnapshotPackageSendError = SendError; pub struct SnapshotPackage { pub root: u64, diff --git a/core/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs similarity index 99% rename from core/src/snapshot_utils.rs rename to ledger/src/snapshot_utils.rs index ca9d0e00bb..970316d577 100644 --- a/core/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -2,6 +2,7 @@ use crate::snapshot_package::SnapshotPackage; use bincode::{deserialize_from, serialize_into}; use bzip2::bufread::BzDecoder; use fs_extra::dir::CopyOptions; +use log::*; use solana_measure::measure::Measure; use solana_runtime::bank::Bank; use solana_runtime::status_cache::SlotDelta; diff --git a/local_cluster/src/tests/local_cluster.rs b/local_cluster/src/tests/local_cluster.rs index 356e99dabd..8a0594bfec 100644 --- a/local_cluster/src/tests/local_cluster.rs +++ b/local_cluster/src/tests/local_cluster.rs @@ -6,10 +6,10 @@ use crate::{ use log::*; use serial_test_derive::serial; use solana_core::{ - bank_forks::SnapshotConfig, broadcast_stage::BroadcastStageType, - gossip_service::discover_cluster, snapshot_utils, validator::ValidatorConfig, + broadcast_stage::BroadcastStageType, gossip_service::discover_cluster, + validator::ValidatorConfig, }; -use solana_ledger::blocktree::Blocktree; +use solana_ledger::{bank_forks::SnapshotConfig, blocktree::Blocktree, snapshot_utils}; use solana_runtime::accounts_db::AccountsDB; use solana_sdk::{ client::SyncClient, diff --git a/validator/src/lib.rs b/validator/src/lib.rs index 72276e10f8..d8ee7a7a16 100644 --- a/validator/src/lib.rs +++ b/validator/src/lib.rs @@ -4,7 +4,6 @@ use console::{style, Emoji}; use indicatif::{ProgressBar, ProgressStyle}; use log::*; use solana_client::rpc_client::RpcClient; -use solana_core::bank_forks::SnapshotConfig; use solana_core::cluster_info::{Node, VALIDATOR_PORT_RANGE}; use solana_core::contact_info::ContactInfo; use solana_core::gossip_service::discover; @@ -12,6 +11,7 @@ use solana_core::ledger_cleanup_service::DEFAULT_MAX_LEDGER_SLOTS; use solana_core::service::Service; use solana_core::socketaddr; use solana_core::validator::{Validator, ValidatorConfig}; +use solana_ledger::bank_forks::SnapshotConfig; use solana_sdk::clock::Slot; use solana_sdk::hash::Hash; use solana_sdk::signature::{read_keypair_file, Keypair, KeypairUtil}; @@ -195,7 +195,7 @@ fn initialize_ledger_path( download_tar_bz2(&rpc_addr, "genesis.tar.bz2", ledger_path, true)?; if !no_snapshot_fetch { - let snapshot_package = solana_core::snapshot_utils::get_snapshot_tar_path(ledger_path); + let snapshot_package = solana_ledger::snapshot_utils::get_snapshot_tar_path(ledger_path); if snapshot_package.exists() { fs::remove_file(&snapshot_package) .unwrap_or_else(|err| warn!("error removing {:?}: {}", snapshot_package, err));