gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rename AccountsDb plugins to Geyser plugins (#23604)

Browse Source
This commit is contained in:
Tyera Eulberg 2022-03-14 19:18:46 -06:00 committed by GitHub
parent bcc5890182
commit 102dd68a03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 238 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
Cargo.lock generated
View File

@ -4221,36 +4221,6 @@ dependencies = [
"spl-token", "spl-token",
] ]
[[package]]
name = "solana-accountsdb-plugin-interface"
version = "1.10.3"
dependencies = [
"log",
"solana-sdk",
"solana-transaction-status",
"thiserror",
]
[[package]]
name = "solana-accountsdb-plugin-manager"
version = "1.10.3"
dependencies = [
"bs58 0.4.0",
"crossbeam-channel",
"json5",
"libloading",
"log",
"serde_json",
"solana-accountsdb-plugin-interface",
"solana-measure",
"solana-metrics",
"solana-rpc",
"solana-runtime",
"solana-sdk",
"solana-transaction-status",
"thiserror",
]
[[package]] [[package]]
name = "solana-address-lookup-table-program" name = "solana-address-lookup-table-program"
version = "1.10.3" version = "1.10.3"
@ -4677,13 +4647,13 @@ dependencies = [
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"serial_test", "serial_test",
"solana-accountsdb-plugin-manager",
"solana-address-lookup-table-program", "solana-address-lookup-table-program",
"solana-bloom", "solana-bloom",
"solana-client", "solana-client",
"solana-entry", "solana-entry",
"solana-frozen-abi 1.10.3", "solana-frozen-abi 1.10.3",
"solana-frozen-abi-macro 1.10.3", "solana-frozen-abi-macro 1.10.3",
"solana-geyser-plugin-manager",
"solana-gossip", "solana-gossip",
"solana-ledger", "solana-ledger",
"solana-logger 1.10.3", "solana-logger 1.10.3",
@ -4892,6 +4862,36 @@ dependencies = [
"solana-sdk", "solana-sdk",
] ]
[[package]]
name = "solana-geyser-plugin-interface"
version = "1.10.3"
dependencies = [
"log",
"solana-sdk",
"solana-transaction-status",
"thiserror",
]
[[package]]
name = "solana-geyser-plugin-manager"
version = "1.10.3"
dependencies = [
"bs58 0.4.0",
"crossbeam-channel",
"json5",
"libloading",
"log",
"serde_json",
"solana-geyser-plugin-interface",
"solana-measure",
"solana-metrics",
"solana-rpc",
"solana-runtime",
"solana-sdk",
"solana-transaction-status",
"thiserror",
]
[[package]] [[package]]
name = "solana-gossip" name = "solana-gossip"
version = "1.10.3" version = "1.10.3"

4
Cargo.toml
View File

@ -3,8 +3,6 @@ members = [
"account-decoder", "account-decoder",
"accounts-bench", "accounts-bench",
"accounts-cluster-bench", "accounts-cluster-bench",
"accountsdb-plugin-interface",
"accountsdb-plugin-manager",
"banking-bench", "banking-bench",
"banks-client", "banks-client",
"banks-interface", "banks-interface",
@ -27,6 +25,8 @@ members = [
"frozen-abi", "frozen-abi",
"genesis", "genesis",
"genesis-utils", "genesis-utils",
"geyser-plugin-interface",
"geyser-plugin-manager",
"gossip", "gossip",
"install", "install",
"keygen", "keygen",

20
accountsdb-plugin-interface/README.md
View File

@ -1,20 +0,0 @@
<p align="center">
<a href="https://solana.com">
<img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
</a>
</p>
# Solana AccountsDb Plugin Interface
This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
instantiates the implementation of the interface.
The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
external PostgreSQL databases.
More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
Still have questions? Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

1
accountsdb-plugin-interface/src/lib.rs
View File

@ -1 +0,0 @@
pub mod accountsdb_plugin_interface;

2
core/Cargo.toml
View File

@ -33,13 +33,13 @@ rayon = "1.5.1"
retain_mut = "0.1.7" retain_mut = "0.1.7"
serde = "1.0.136" serde = "1.0.136"
serde_derive = "1.0.103" serde_derive = "1.0.103"
solana-accountsdb-plugin-manager = { path = "../accountsdb-plugin-manager", version = "=1.10.3" }
solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.10.3" } solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.10.3" }
solana-bloom = { path = "../bloom", version = "=1.10.3" } solana-bloom = { path = "../bloom", version = "=1.10.3" }
solana-client = { path = "../client", version = "=1.10.3" } solana-client = { path = "../client", version = "=1.10.3" }
solana-entry = { path = "../entry", version = "=1.10.3" } solana-entry = { path = "../entry", version = "=1.10.3" }
solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.3" } solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.3" }
solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.3" } solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.3" }
solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "=1.10.3" }
solana-gossip = { path = "../gossip", version = "=1.10.3" } solana-gossip = { path = "../gossip", version = "=1.10.3" }
solana-ledger = { path = "../ledger", version = "=1.10.3" } solana-ledger = { path = "../ledger", version = "=1.10.3" }
solana-measure = { path = "../measure", version = "=1.10.3" } solana-measure = { path = "../measure", version = "=1.10.3" }

2
core/src/replay_stage.rs
View File

@ -27,9 +27,9 @@ use {
window_service::DuplicateSlotReceiver, window_service::DuplicateSlotReceiver,
}, },
crossbeam_channel::{Receiver, RecvTimeoutError, Sender}, crossbeam_channel::{Receiver, RecvTimeoutError, Sender},
solana_accountsdb_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
solana_client::rpc_response::SlotUpdate, solana_client::rpc_response::SlotUpdate,
solana_entry::entry::VerifyRecyclers, solana_entry::entry::VerifyRecyclers,
solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
solana_gossip::cluster_info::ClusterInfo, solana_gossip::cluster_info::ClusterInfo,
solana_ledger::{ solana_ledger::{
block_error::BlockError, block_error::BlockError,

2
core/src/tvu.rs
View File

@ -27,7 +27,7 @@ use {
voting_service::VotingService, voting_service::VotingService,
}, },
crossbeam_channel::{unbounded, Receiver}, crossbeam_channel::{unbounded, Receiver},
solana_accountsdb_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock, solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
solana_gossip::cluster_info::ClusterInfo, solana_gossip::cluster_info::ClusterInfo,
solana_ledger::{ solana_ledger::{
blockstore::Blockstore, blockstore_processor::TransactionStatusSender, blockstore::Blockstore, blockstore_processor::TransactionStatusSender,

59
core/src/validator.rs
View File

@ -22,8 +22,8 @@ use {
}, },
crossbeam_channel::{bounded, unbounded, Receiver}, crossbeam_channel::{bounded, unbounded, Receiver},
rand::{thread_rng, Rng}, rand::{thread_rng, Rng},
solana_accountsdb_plugin_manager::accountsdb_plugin_service::AccountsDbPluginService,
solana_entry::poh::compute_hash_time_ns, solana_entry::poh::compute_hash_time_ns,
solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService,
solana_gossip::{ solana_gossip::{
cluster_info::{ cluster_info::{
ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS, ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
@ -119,7 +119,7 @@ pub struct ValidatorConfig {
pub account_shrink_paths: Option<Vec<PathBuf>>, pub account_shrink_paths: Option<Vec<PathBuf>>,
pub rpc_config: JsonRpcConfig, pub rpc_config: JsonRpcConfig,
pub accountsdb_repl_service_config: Option<AccountsDbReplServiceConfig>, pub accountsdb_repl_service_config: Option<AccountsDbReplServiceConfig>,
pub accountsdb_plugin_config_files: Option<Vec<PathBuf>>, pub geyser_plugin_config_files: Option<Vec<PathBuf>>,
pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub) pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub)
pub pubsub_config: PubSubConfig, pub pubsub_config: PubSubConfig,
pub snapshot_config: Option<SnapshotConfig>, pub snapshot_config: Option<SnapshotConfig>,
@ -181,7 +181,7 @@ impl Default for ValidatorConfig {
account_shrink_paths: None, account_shrink_paths: None,
rpc_config: JsonRpcConfig::default(), rpc_config: JsonRpcConfig::default(),
accountsdb_repl_service_config: None, accountsdb_repl_service_config: None,
accountsdb_plugin_config_files: None, geyser_plugin_config_files: None,
rpc_addrs: None, rpc_addrs: None,
pubsub_config: PubSubConfig::default(), pubsub_config: PubSubConfig::default(),
snapshot_config: None, snapshot_config: None,
@ -301,7 +301,7 @@ pub struct Validator {
pub cluster_info: Arc<ClusterInfo>, pub cluster_info: Arc<ClusterInfo>,
pub bank_forks: Arc<RwLock<BankForks>>, pub bank_forks: Arc<RwLock<BankForks>>,
accountsdb_repl_service: Option<AccountsDbReplService>, accountsdb_repl_service: Option<AccountsDbReplService>,
accountsdb_plugin_service: Option<AccountsDbPluginService>, geyser_plugin_service: Option<GeyserPluginService>,
} }
// in the distant future, get rid of ::new()/exit() and use Result properly... // in the distant future, get rid of ::new()/exit() and use Result properly...
@ -340,18 +340,16 @@ impl Validator {
let mut bank_notification_senders = Vec::new(); let mut bank_notification_senders = Vec::new();
let accountsdb_plugin_service = let geyser_plugin_service =
if let Some(accountsdb_plugin_config_files) = &config.accountsdb_plugin_config_files { if let Some(geyser_plugin_config_files) = &config.geyser_plugin_config_files {
let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded(); let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded();
bank_notification_senders.push(confirmed_bank_sender); bank_notification_senders.push(confirmed_bank_sender);
let result = AccountsDbPluginService::new( let result =
confirmed_bank_receiver, GeyserPluginService::new(confirmed_bank_receiver, geyser_plugin_config_files);
accountsdb_plugin_config_files,
);
match result { match result {
Ok(accountsdb_plugin_service) => Some(accountsdb_plugin_service), Ok(geyser_plugin_service) => Some(geyser_plugin_service),
Err(err) => { Err(err) => {
error!("Failed to load the AccountsDb plugin: {:?}", err); error!("Failed to load the Geyser plugin: {:?}", err);
abort(); abort();
} }
} }
@ -425,29 +423,20 @@ impl Validator {
let accounts_package_channel = unbounded(); let accounts_package_channel = unbounded();
let accounts_update_notifier = let accounts_update_notifier = geyser_plugin_service
accountsdb_plugin_service .as_ref()
.as_ref() .and_then(|geyser_plugin_service| geyser_plugin_service.get_accounts_update_notifier());
.and_then(|accountsdb_plugin_service| {
accountsdb_plugin_service.get_accounts_update_notifier()
});
let transaction_notifier = let transaction_notifier = geyser_plugin_service
accountsdb_plugin_service .as_ref()
.as_ref() .and_then(|geyser_plugin_service| geyser_plugin_service.get_transaction_notifier());
.and_then(|accountsdb_plugin_service| {
accountsdb_plugin_service.get_transaction_notifier()
});
let block_metadata_notifier = let block_metadata_notifier = geyser_plugin_service
accountsdb_plugin_service .as_ref()
.as_ref() .and_then(|geyser_plugin_service| geyser_plugin_service.get_block_metadata_notifier());
.and_then(|accountsdb_plugin_service| {
accountsdb_plugin_service.get_block_metadata_notifier()
});
info!( info!(
"AccountsDb plugin: accounts_update_notifier: {} transaction_notifier: {}", "Geyser plugin: accounts_update_notifier: {} transaction_notifier: {}",
accounts_update_notifier.is_some(), accounts_update_notifier.is_some(),
transaction_notifier.is_some() transaction_notifier.is_some()
); );
@ -949,7 +938,7 @@ impl Validator {
cluster_info, cluster_info,
bank_forks, bank_forks,
accountsdb_repl_service, accountsdb_repl_service,
accountsdb_plugin_service, geyser_plugin_service,
} }
} }
@ -1071,10 +1060,8 @@ impl Validator {
.expect("accountsdb_repl_service"); .expect("accountsdb_repl_service");
} }
if let Some(accountsdb_plugin_service) = self.accountsdb_plugin_service { if let Some(geyser_plugin_service) = self.geyser_plugin_service {
accountsdb_plugin_service geyser_plugin_service.join().expect("geyser_plugin_service");
.join()
.expect("accountsdb_plugin_service");
} }
} }
} }

2
docs/sidebars.js
View File

@ -80,7 +80,7 @@ module.exports = {
}, },
"developing/test-validator", "developing/test-validator",
"developing/backwards-compatibility", "developing/backwards-compatibility",
"developing/plugins/accountsdb_plugin" "developing/plugins/geyser-plugins"
], ],
Integrating: ["integrations/exchange"], Integrating: ["integrations/exchange"],
Validating: [ Validating: [

2
docs/src/developing/backwards-compatibility.md
View File

@ -76,7 +76,7 @@ Major releases:
- [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs - [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs
- [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API - [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API
- [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files - [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files
- [`solana-accountsdb-plugin-interface`](https://docs.rs/solana-accountsdb-plugin-interface/) - Rust interface for developing Solana AccountsDb plugins. - [`solana-geyser-plugin-interface`](https://docs.rs/solana-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins.
Patch releases: Patch releases:

44
docs/src/developing/plugins/accountsdb_plugin.md → docs/src/developing/plugins/geyser-plugins.md
View File

@ -1,18 +1,18 @@
--- ---
title: Plugins title: Geyser Plugins
--- ---
## Overview ## Overview
Validators under heavy RPC loads, such as when serving getProgramAccounts calls, Validators under heavy RPC loads, such as when serving getProgramAccounts calls,
can fall behind the network. To solve this problem, the validator has been can fall behind the network. To solve this problem, the validator has been
enhanced to support a plugin mechanism through which the information about enhanced to support a plugin mechanism, called a "Geyser" plugin, through which
accounts and slots can be transmitted to external data stores such as relational the information about accounts, slots, blocks, and transactions can be
databases, NoSQL databases or Kafka. RPC services then can be developed to transmitted to external data stores such as relational databases, NoSQL
consume data from these external data stores with the possibility of more databases or Kafka. RPC services then can be developed to consume data from
flexible and targeted optimizations such as caching and indexing. This allows these external data stores with the possibility of more flexible and targeted
the validator to focus on processing transactions without being slowed down by optimizations such as caching and indexing. This allows the validator to focus
busy RPC requests. on processing transactions without being slowed down by busy RPC requests.
This document describes the interfaces of the plugin and the referential plugin This document describes the interfaces of the plugin and the referential plugin
implementation for the PostgreSQL database. implementation for the PostgreSQL database.
@ -22,24 +22,24 @@ implementation for the PostgreSQL database.
### Important Crates: ### Important Crates:
- [`solana-accountsdb-plugin-interface`] &mdash; This crate defines the plugin - [`solana-geyser-plugin-interface`] &mdash; This crate defines the plugin
interfaces. interfaces.
- [`solana-accountsdb-plugin-postgres`] &mdash; The crate for the referential - [`solana-accountsdb-plugin-postgres`] &mdash; The crate for the referential
plugin implementation for the PostgreSQL database. plugin implementation for the PostgreSQL database.
[`solana-accountsdb-plugin-interface`]: https://docs.rs/solana-accountsdb-plugin-interface [`solana-geyser-plugin-interface`]: https://docs.rs/solana-geyser-plugin-interface
[`solana-accountsdb-plugin-postgres`]: https://docs.rs/solana-accountsdb-plugin-postgres [`solana-accountsdb-plugin-postgres`]: https://docs.rs/solana-accountsdb-plugin-postgres
[`solana-sdk`]: https://docs.rs/solana-sdk [`solana-sdk`]: https://docs.rs/solana-sdk
[`solana-transaction-status`]: https://docs.rs/solana-transaction-status [`solana-transaction-status`]: https://docs.rs/solana-transaction-status
## The Plugin Interface ## The Plugin Interface
The Plugin interface is declared in [`solana-accountsdb-plugin-interface`]. It The Plugin interface is declared in [`solana-geyser-plugin-interface`]. It
is defined by the trait `AccountsDbPlugin`. The plugin should implement the is defined by the trait `GeyserPlugin`. The plugin should implement the
trait and expose a "C" function `_create_plugin` to return the pointer to this trait and expose a "C" function `_create_plugin` to return the pointer to this
trait. For example, in the referential implementation, the following code trait. For example, in the referential implementation, the following code
instantiates the PostgreSQL plugin `AccountsDbPluginPostgres ` and returns its instantiates the PostgreSQL plugin `GeyserPluginPostgres ` and returns its
pointer. pointer.
``` ```
@ -47,10 +47,10 @@ pointer.
#[allow(improper_ctypes_definitions)] #[allow(improper_ctypes_definitions)]
/// # Safety /// # Safety
/// ///
/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin. /// This function returns the GeyserPluginPostgres pointer as trait GeyserPlugin.
pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin { pub unsafe extern "C" fn _create_plugin() -> *mut dyn GeyserPlugin {
let plugin = AccountsDbPluginPostgres::new(); let plugin = GeyserPluginPostgres::new();
let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin); let plugin: Box<dyn GeyserPlugin> = Box::new(plugin);
Box::into_raw(plugin) Box::into_raw(plugin)
} }
``` ```
@ -62,7 +62,7 @@ file in JSON5 format. The JSON5 file must have a field `libpath` that points
to the full path name of the shared library implementing the plugin, and may to the full path name of the shared library implementing the plugin, and may
have other configuration information, like connection parameters for the external have other configuration information, like connection parameters for the external
database. The plugin configuration file is specified by the validator's CLI database. The plugin configuration file is specified by the validator's CLI
parameter `--accountsdb-plugin-config` and the file must be readable to the parameter `--geyser-plugin-config` and the file must be readable to the
validator process. validator process.
Please see the [config file](#config) for the referential Please see the [config file](#config) for the referential
@ -164,11 +164,11 @@ please refer to [`solana-sdk`] and [`solana-transaction-status`]
The `slot` points to the slot the transaction is executed at. The `slot` points to the slot the transaction is executed at.
For more details, please refer to the Rust documentation in For more details, please refer to the Rust documentation in
[`solana-accountsdb-plugin-interface`]. [`solana-geyser-plugin-interface`].
## Example PostgreSQL Plugin ## Example PostgreSQL Plugin
The [`solana-accountsdb-plugin-postgres`] crate implements a plugin storing The [`solana-accountsdb-plugin-postgres`] repository implements a plugin storing
account data to a PostgreSQL database to illustrate how a plugin can be account data to a PostgreSQL database to illustrate how a plugin can be
developed. developed.
@ -182,7 +182,7 @@ configuration file looks like the following:
``` ```
{ {
"libpath": "/solana/target/release/libsolana_accountsdb_plugin_postgres.so", "libpath": "/solana/target/release/libsolana_geyser_plugin_postgres.so",
"host": "postgres-server", "host": "postgres-server",
"user": "solana", "user": "solana",
"port": 5433, "port": 5433,
@ -366,7 +366,7 @@ Then run the script:
psql -U solana -p 5433 -h 10.138.0.9 -w -d solana -f create_schema.sql psql -U solana -p 5433 -h 10.138.0.9 -w -d solana -f create_schema.sql
``` ```
After this, start the validator with the plugin by using the `--accountsdb-plugin-config` After this, start the validator with the plugin by using the `--geyser-plugin-config`
argument mentioned above. argument mentioned above.
#### Destroy the Schema Objects #### Destroy the Schema Objects

6
accountsdb-plugin-interface/Cargo.toml → geyser-plugin-interface/Cargo.toml
View File

@ -1,13 +1,13 @@
[package] [package]
authors = ["Solana Maintainers <maintainers@solana.foundation>"] authors = ["Solana Maintainers <maintainers@solana.foundation>"]
edition = "2021" edition = "2021"
name = "solana-accountsdb-plugin-interface" name = "solana-geyser-plugin-interface"
description = "The Solana AccountsDb plugin interface." description = "The Solana Geyser plugin interface."
version = "1.10.3" version = "1.10.3"
repository = "https://github.com/solana-labs/solana" repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0" license = "Apache-2.0"
homepage = "https://solana.com/" homepage = "https://solana.com/"
documentation = "https://docs.rs/solana-accountsdb-plugin-interface" documentation = "https://docs.rs/solana-geyser-plugin-interface"
[dependencies] [dependencies]
log = "0.4.11" log = "0.4.11"

25
geyser-plugin-interface/README.md Normal file
View File

@ -0,0 +1,25 @@
<p align="center">
<a href="https://solana.com">
<img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
</a>
</p>
# Solana Geyser Plugin Interface
This crate enables an plugin to be added into the Solana Validator runtime to
take actions at the time of account updates or block and transaction processing;
for example, saving the account state to an external database. The plugin must
implement the `GeyserPlugin` trait. Please see the detail of the
`geyser_plugin_interface.rs` for the interface definition.
The plugin should produce a `cdylib` dynamic library, which must expose a `C`
function `_create_plugin()` that instantiates the implementation of the
interface.
The https://github.com/solana-labs/solana-accountsdb-plugin-postgres repository
provides an example of how to create a plugin which saves the accounts data into
an external PostgreSQL databases.
More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
Still have questions? Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

14
accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs → geyser-plugin-interface/src/geyser_plugin_interface.rs
View File

@ -1,5 +1,5 @@
/// The interface for AccountsDb plugins. A plugin must implement /// The interface for Geyser plugins. A plugin must implement
/// the AccountsDbPlugin trait to work with the runtime. /// the GeyserPlugin trait to work with the runtime.
/// In addition, the dynamic library must export a "C" function _create_plugin which /// In addition, the dynamic library must export a "C" function _create_plugin which
/// creates the implementation of the plugin. /// creates the implementation of the plugin.
use { use {
@ -85,7 +85,7 @@ pub enum ReplicaBlockInfoVersions<'a> {
/// Errors returned by plugin calls /// Errors returned by plugin calls
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum AccountsDbPluginError { pub enum GeyserPluginError {
/// Error opening the configuration file; for example, when the file /// Error opening the configuration file; for example, when the file
/// is not found or when the validator process has no permission to read it. /// is not found or when the validator process has no permission to read it.
#[error("Error opening config file. Error detail: ({0}).")] #[error("Error opening config file. Error detail: ({0}).")]
@ -134,12 +134,12 @@ impl SlotStatus {
} }
} }
pub type Result<T> = std::result::Result<T, AccountsDbPluginError>; pub type Result<T> = std::result::Result<T, GeyserPluginError>;
/// Defines an AccountsDb plugin, to stream data from the runtime. /// Defines a Geyser plugin, to stream data from the runtime.
/// AccountsDb plugins must describe desired behavior for load and unload, /// Geyser plugins must describe desired behavior for load and unload,
/// as well as how they will handle streamed data. /// as well as how they will handle streamed data.
pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug { pub trait GeyserPlugin: Any + Send + Sync + std::fmt::Debug {
fn name(&self) -> &'static str; fn name(&self) -> &'static str;
/// The callback called when a plugin is loaded by the system, /// The callback called when a plugin is loaded by the system,

1
geyser-plugin-interface/src/lib.rs Normal file
View File

@ -0,0 +1 @@
pub mod geyser_plugin_interface;

8
accountsdb-plugin-manager/Cargo.toml → geyser-plugin-manager/Cargo.toml
View File

@ -1,13 +1,13 @@
[package] [package]
authors = ["Solana Maintainers <maintainers@solana.foundation>"] authors = ["Solana Maintainers <maintainers@solana.foundation>"]
edition = "2021" edition = "2021"
name = "solana-accountsdb-plugin-manager" name = "solana-geyser-plugin-manager"
description = "The Solana AccountsDb plugin manager." description = "The Solana Geyser plugin manager."
version = "1.10.3" version = "1.10.3"
repository = "https://github.com/solana-labs/solana" repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0" license = "Apache-2.0"
homepage = "https://solana.com/" homepage = "https://solana.com/"
documentation = "https://docs.rs/solana-validator" documentation = "https://docs.rs/solana-geyser-plugin-manager"
[dependencies] [dependencies]
bs58 = "0.4.0" bs58 = "0.4.0"
@ -16,7 +16,7 @@ json5 = "0.4.1"
libloading = "0.7.3" libloading = "0.7.3"
log = "0.4.11" log = "0.4.11"
serde_json = "1.0.79" serde_json = "1.0.79"
solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.3" } solana-geyser-plugin-interface = { path = "../geyser-plugin-interface", version = "=1.10.3" }
solana-measure = { path = "../measure", version = "=1.10.3" } solana-measure = { path = "../measure", version = "=1.10.3" }
solana-metrics = { path = "../metrics", version = "=1.10.3" } solana-metrics = { path = "../metrics", version = "=1.10.3" }
solana-rpc = { path = "../rpc", version = "=1.10.3" } solana-rpc = { path = "../rpc", version = "=1.10.3" }

28
accountsdb-plugin-manager/src/accounts_update_notifier.rs → geyser-plugin-manager/src/accounts_update_notifier.rs
View File

@ -1,8 +1,8 @@
/// Module responsible for notifying plugins of account updates /// Module responsible for notifying plugins of account updates
use { use {
crate::accountsdb_plugin_manager::AccountsDbPluginManager, crate::geyser_plugin_manager::GeyserPluginManager,
log::*, log::*,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{ solana_geyser_plugin_interface::geyser_plugin_interface::{
ReplicaAccountInfo, ReplicaAccountInfoVersions, ReplicaAccountInfo, ReplicaAccountInfoVersions,
}, },
solana_measure::measure::Measure, solana_measure::measure::Measure,
@ -19,7 +19,7 @@ use {
}; };
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct AccountsUpdateNotifierImpl { pub(crate) struct AccountsUpdateNotifierImpl {
plugin_manager: Arc<RwLock<AccountsDbPluginManager>>, plugin_manager: Arc<RwLock<GeyserPluginManager>>,
} }
impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl { impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
@ -30,14 +30,14 @@ impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
} }
fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) { fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) {
let mut measure_all = Measure::start("accountsdb-plugin-notify-account-restore-all"); let mut measure_all = Measure::start("geyser-plugin-notify-account-restore-all");
let mut measure_copy = Measure::start("accountsdb-plugin-copy-stored-account-info"); let mut measure_copy = Measure::start("geyser-plugin-copy-stored-account-info");
let account = self.accountinfo_from_stored_account_meta(account); let account = self.accountinfo_from_stored_account_meta(account);
measure_copy.stop(); measure_copy.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-copy-stored-account-info-us", "geyser-plugin-copy-stored-account-info-us",
measure_copy.as_us() as usize, measure_copy.as_us() as usize,
100000, 100000,
100000 100000
@ -49,7 +49,7 @@ impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
measure_all.stop(); measure_all.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-notify-account-restore-all-us", "geyser-plugin-notify-account-restore-all-us",
measure_all.as_us() as usize, measure_all.as_us() as usize,
100000, 100000,
100000 100000
@ -63,7 +63,7 @@ impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
} }
for plugin in plugin_manager.plugins.iter_mut() { for plugin in plugin_manager.plugins.iter_mut() {
let mut measure = Measure::start("accountsdb-plugin-end-of-restore-from-snapshot"); let mut measure = Measure::start("geyser-plugin-end-of-restore-from-snapshot");
match plugin.notify_end_of_startup() { match plugin.notify_end_of_startup() {
Err(err) => { Err(err) => {
error!( error!(
@ -81,7 +81,7 @@ impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
} }
measure.stop(); measure.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-end-of-restore-from-snapshot", "geyser-plugin-end-of-restore-from-snapshot",
measure.as_us() as usize measure.as_us() as usize
); );
} }
@ -89,7 +89,7 @@ impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
} }
impl AccountsUpdateNotifierImpl { impl AccountsUpdateNotifierImpl {
pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self { pub fn new(plugin_manager: Arc<RwLock<GeyserPluginManager>>) -> Self {
AccountsUpdateNotifierImpl { plugin_manager } AccountsUpdateNotifierImpl { plugin_manager }
} }
@ -130,14 +130,14 @@ impl AccountsUpdateNotifierImpl {
slot: Slot, slot: Slot,
is_startup: bool, is_startup: bool,
) { ) {
let mut measure2 = Measure::start("accountsdb-plugin-notify_plugins_of_account_update"); let mut measure2 = Measure::start("geyser-plugin-notify_plugins_of_account_update");
let mut plugin_manager = self.plugin_manager.write().unwrap(); let mut plugin_manager = self.plugin_manager.write().unwrap();
if plugin_manager.plugins.is_empty() { if plugin_manager.plugins.is_empty() {
return; return;
} }
for plugin in plugin_manager.plugins.iter_mut() { for plugin in plugin_manager.plugins.iter_mut() {
let mut measure = Measure::start("accountsdb-plugin-update-account"); let mut measure = Measure::start("geyser-plugin-update-account");
match plugin.update_account( match plugin.update_account(
ReplicaAccountInfoVersions::V0_0_1(&account), ReplicaAccountInfoVersions::V0_0_1(&account),
slot, slot,
@ -163,7 +163,7 @@ impl AccountsUpdateNotifierImpl {
} }
measure.stop(); measure.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-update-account-us", "geyser-plugin-update-account-us",
measure.as_us() as usize, measure.as_us() as usize,
100000, 100000,
100000 100000
@ -171,7 +171,7 @@ impl AccountsUpdateNotifierImpl {
} }
measure2.stop(); measure2.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-notify_plugins_of_account_update-us", "geyser-plugin-notify_plugins_of_account_update-us",
measure2.as_us() as usize, measure2.as_us() as usize,
100000, 100000,
100000 100000

12
accountsdb-plugin-manager/src/block_metadata_notifier.rs → geyser-plugin-manager/src/block_metadata_notifier.rs
View File

@ -1,10 +1,10 @@
use { use {
crate::{ crate::{
accountsdb_plugin_manager::AccountsDbPluginManager,
block_metadata_notifier_interface::BlockMetadataNotifier, block_metadata_notifier_interface::BlockMetadataNotifier,
geyser_plugin_manager::GeyserPluginManager,
}, },
log::*, log::*,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{ solana_geyser_plugin_interface::geyser_plugin_interface::{
ReplicaBlockInfo, ReplicaBlockInfoVersions, ReplicaBlockInfo, ReplicaBlockInfoVersions,
}, },
solana_measure::measure::Measure, solana_measure::measure::Measure,
@ -16,7 +16,7 @@ use {
}; };
pub(crate) struct BlockMetadataNotifierImpl { pub(crate) struct BlockMetadataNotifierImpl {
plugin_manager: Arc<RwLock<AccountsDbPluginManager>>, plugin_manager: Arc<RwLock<GeyserPluginManager>>,
} }
impl BlockMetadataNotifier for BlockMetadataNotifierImpl { impl BlockMetadataNotifier for BlockMetadataNotifierImpl {
@ -36,7 +36,7 @@ impl BlockMetadataNotifier for BlockMetadataNotifierImpl {
let rewards = Self::build_rewards(rewards); let rewards = Self::build_rewards(rewards);
for plugin in plugin_manager.plugins.iter_mut() { for plugin in plugin_manager.plugins.iter_mut() {
let mut measure = Measure::start("accountsdb-plugin-update-slot"); let mut measure = Measure::start("geyser-plugin-update-slot");
let block_info = let block_info =
Self::build_replica_block_info(slot, blockhash, &rewards, block_time, block_height); Self::build_replica_block_info(slot, blockhash, &rewards, block_time, block_height);
let block_info = ReplicaBlockInfoVersions::V0_0_1(&block_info); let block_info = ReplicaBlockInfoVersions::V0_0_1(&block_info);
@ -59,7 +59,7 @@ impl BlockMetadataNotifier for BlockMetadataNotifierImpl {
} }
measure.stop(); measure.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-update-block-metadata-us", "geyser-plugin-update-block-metadata-us",
measure.as_us() as usize, measure.as_us() as usize,
1000, 1000,
1000 1000
@ -99,7 +99,7 @@ impl BlockMetadataNotifierImpl {
} }
} }
pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self { pub fn new(plugin_manager: Arc<RwLock<GeyserPluginManager>>) -> Self {
Self { plugin_manager } Self { plugin_manager }
} }
} }

0
accountsdb-plugin-manager/src/block_metadata_notifier_interface.rs → geyser-plugin-manager/src/block_metadata_notifier_interface.rs
View File

14
accountsdb-plugin-manager/src/accountsdb_plugin_manager.rs → geyser-plugin-manager/src/geyser_plugin_manager.rs
View File

@ -1,20 +1,20 @@
/// Managing the AccountsDb plugins /// Managing the Geyser plugins
use { use {
libloading::{Library, Symbol}, libloading::{Library, Symbol},
log::*, log::*,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::AccountsDbPlugin, solana_geyser_plugin_interface::geyser_plugin_interface::GeyserPlugin,
std::error::Error, std::error::Error,
}; };
#[derive(Default, Debug)] #[derive(Default, Debug)]
pub struct AccountsDbPluginManager { pub struct GeyserPluginManager {
pub plugins: Vec<Box<dyn AccountsDbPlugin>>, pub plugins: Vec<Box<dyn GeyserPlugin>>,
libs: Vec<Library>, libs: Vec<Library>,
} }
impl AccountsDbPluginManager { impl GeyserPluginManager {
pub fn new() -> Self { pub fn new() -> Self {
AccountsDbPluginManager { GeyserPluginManager {
plugins: Vec::default(), plugins: Vec::default(),
libs: Vec::default(), libs: Vec::default(),
} }
@ -29,7 +29,7 @@ impl AccountsDbPluginManager {
libpath: &str, libpath: &str,
config_file: &str, config_file: &str,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
type PluginConstructor = unsafe fn() -> *mut dyn AccountsDbPlugin; type PluginConstructor = unsafe fn() -> *mut dyn GeyserPlugin;
let lib = Library::new(libpath)?; let lib = Library::new(libpath)?;
let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?; let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?;
let plugin_raw = constructor(); let plugin_raw = constructor();

81
accountsdb-plugin-manager/src/accountsdb_plugin_service.rs → geyser-plugin-manager/src/geyser_plugin_service.rs
View File

@ -1,11 +1,10 @@
use { use {
crate::{ crate::{
accounts_update_notifier::AccountsUpdateNotifierImpl, accounts_update_notifier::AccountsUpdateNotifierImpl,
accountsdb_plugin_manager::AccountsDbPluginManager,
block_metadata_notifier::BlockMetadataNotifierImpl, block_metadata_notifier::BlockMetadataNotifierImpl,
block_metadata_notifier_interface::BlockMetadataNotifierLock, block_metadata_notifier_interface::BlockMetadataNotifierLock,
slot_status_notifier::SlotStatusNotifierImpl, slot_status_observer::SlotStatusObserver, geyser_plugin_manager::GeyserPluginManager, slot_status_notifier::SlotStatusNotifierImpl,
transaction_notifier::TransactionNotifierImpl, slot_status_observer::SlotStatusObserver, transaction_notifier::TransactionNotifierImpl,
}, },
crossbeam_channel::Receiver, crossbeam_channel::Receiver,
log::*, log::*,
@ -25,7 +24,7 @@ use {
}; };
#[derive(Error, Debug)] #[derive(Error, Debug)]
pub enum AccountsdbPluginServiceError { pub enum GeyserPluginServiceError {
#[error("Cannot open the the plugin config file")] #[error("Cannot open the the plugin config file")]
CannotOpenConfigFile(String), CannotOpenConfigFile(String),
@ -45,41 +44,41 @@ pub enum AccountsdbPluginServiceError {
PluginLoadError(String), PluginLoadError(String),
} }
/// The service managing the AccountsDb plugin workflow. /// The service managing the Geyser plugin workflow.
pub struct AccountsDbPluginService { pub struct GeyserPluginService {
slot_status_observer: Option<SlotStatusObserver>, slot_status_observer: Option<SlotStatusObserver>,
plugin_manager: Arc<RwLock<AccountsDbPluginManager>>, plugin_manager: Arc<RwLock<GeyserPluginManager>>,
accounts_update_notifier: Option<AccountsUpdateNotifier>, accounts_update_notifier: Option<AccountsUpdateNotifier>,
transaction_notifier: Option<TransactionNotifierLock>, transaction_notifier: Option<TransactionNotifierLock>,
block_metadata_notifier: Option<BlockMetadataNotifierLock>, block_metadata_notifier: Option<BlockMetadataNotifierLock>,
} }
impl AccountsDbPluginService { impl GeyserPluginService {
/// Creates and returns the AccountsDbPluginService. /// Creates and returns the GeyserPluginService.
/// # Arguments /// # Arguments
/// * `confirmed_bank_receiver` - The receiver for confirmed bank notification /// * `confirmed_bank_receiver` - The receiver for confirmed bank notification
/// * `accountsdb_plugin_config_file` - The config file path for the plugin. The /// * `geyser_plugin_config_file` - The config file path for the plugin. The
/// config file controls the plugin responsible /// config file controls the plugin responsible
/// for transporting the data to external data stores. It is defined in JSON format. /// for transporting the data to external data stores. It is defined in JSON format.
/// The `libpath` field should be pointed to the full path of the dynamic shared library /// The `libpath` field should be pointed to the full path of the dynamic shared library
/// (.so file) to be loaded. The shared library must implement the `AccountsDbPlugin` /// (.so file) to be loaded. The shared library must implement the `GeyserPlugin`
/// trait. And the shared library shall export a `C` function `_create_plugin` which /// trait. And the shared library shall export a `C` function `_create_plugin` which
/// shall create the implementation of `AccountsDbPlugin` and returns to the caller. /// shall create the implementation of `GeyserPlugin` and returns to the caller.
/// The rest of the JSON fields' definition is up to to the concrete plugin implementation /// The rest of the JSON fields' definition is up to to the concrete plugin implementation
/// It is usually used to configure the connection information for the external data store. /// It is usually used to configure the connection information for the external data store.
pub fn new( pub fn new(
confirmed_bank_receiver: Receiver<BankNotification>, confirmed_bank_receiver: Receiver<BankNotification>,
accountsdb_plugin_config_files: &[PathBuf], geyser_plugin_config_files: &[PathBuf],
) -> Result<Self, AccountsdbPluginServiceError> { ) -> Result<Self, GeyserPluginServiceError> {
info!( info!(
"Starting AccountsDbPluginService from config files: {:?}", "Starting GeyserPluginService from config files: {:?}",
accountsdb_plugin_config_files geyser_plugin_config_files
); );
let mut plugin_manager = AccountsDbPluginManager::new(); let mut plugin_manager = GeyserPluginManager::new();
for accountsdb_plugin_config_file in accountsdb_plugin_config_files { for geyser_plugin_config_file in geyser_plugin_config_files {
Self::load_plugin(&mut plugin_manager, accountsdb_plugin_config_file)?; Self::load_plugin(&mut plugin_manager, geyser_plugin_config_file)?;
} }
let account_data_notifications_enabled = let account_data_notifications_enabled =
plugin_manager.account_data_notifications_enabled(); plugin_manager.account_data_notifications_enabled();
@ -123,8 +122,8 @@ impl AccountsDbPluginService {
(None, None) (None, None)
}; };
info!("Started AccountsDbPluginService"); info!("Started GeyserPluginService");
Ok(AccountsDbPluginService { Ok(GeyserPluginService {
slot_status_observer, slot_status_observer,
plugin_manager, plugin_manager,
accounts_update_notifier, accounts_update_notifier,
@ -134,57 +133,55 @@ impl AccountsDbPluginService {
} }
fn load_plugin( fn load_plugin(
plugin_manager: &mut AccountsDbPluginManager, plugin_manager: &mut GeyserPluginManager,
accountsdb_plugin_config_file: &Path, geyser_plugin_config_file: &Path,
) -> Result<(), AccountsdbPluginServiceError> { ) -> Result<(), GeyserPluginServiceError> {
let mut file = match File::open(accountsdb_plugin_config_file) { let mut file = match File::open(geyser_plugin_config_file) {
Ok(file) => file, Ok(file) => file,
Err(err) => { Err(err) => {
return Err(AccountsdbPluginServiceError::CannotOpenConfigFile(format!( return Err(GeyserPluginServiceError::CannotOpenConfigFile(format!(
"Failed to open the plugin config file {:?}, error: {:?}", "Failed to open the plugin config file {:?}, error: {:?}",
accountsdb_plugin_config_file, err geyser_plugin_config_file, err
))); )));
} }
}; };
let mut contents = String::new(); let mut contents = String::new();
if let Err(err) = file.read_to_string(&mut contents) { if let Err(err) = file.read_to_string(&mut contents) {
return Err(AccountsdbPluginServiceError::CannotReadConfigFile(format!( return Err(GeyserPluginServiceError::CannotReadConfigFile(format!(
"Failed to read the plugin config file {:?}, error: {:?}", "Failed to read the plugin config file {:?}, error: {:?}",
accountsdb_plugin_config_file, err geyser_plugin_config_file, err
))); )));
} }
let result: serde_json::Value = match json5::from_str(&contents) { let result: serde_json::Value = match json5::from_str(&contents) {
Ok(value) => value, Ok(value) => value,
Err(err) => { Err(err) => {
return Err(AccountsdbPluginServiceError::InvalidConfigFileFormat( return Err(GeyserPluginServiceError::InvalidConfigFileFormat(format!(
format!( "The config file {:?} is not in a valid Json5 format, error: {:?}",
"The config file {:?} is not in a valid Json5 format, error: {:?}", geyser_plugin_config_file, err
accountsdb_plugin_config_file, err )));
),
));
} }
}; };
let libpath = result["libpath"] let libpath = result["libpath"]
.as_str() .as_str()
.ok_or(AccountsdbPluginServiceError::LibPathNotSet)?; .ok_or(GeyserPluginServiceError::LibPathNotSet)?;
let mut libpath = PathBuf::from(libpath); let mut libpath = PathBuf::from(libpath);
if libpath.is_relative() { if libpath.is_relative() {
let config_dir = accountsdb_plugin_config_file.parent().ok_or_else(|| { let config_dir = geyser_plugin_config_file.parent().ok_or_else(|| {
AccountsdbPluginServiceError::CannotOpenConfigFile(format!( GeyserPluginServiceError::CannotOpenConfigFile(format!(
"Failed to resolve parent of {:?}", "Failed to resolve parent of {:?}",
accountsdb_plugin_config_file, geyser_plugin_config_file,
)) ))
})?; })?;
libpath = config_dir.join(libpath); libpath = config_dir.join(libpath);
} }
let config_file = accountsdb_plugin_config_file let config_file = geyser_plugin_config_file
.as_os_str() .as_os_str()
.to_str() .to_str()
.ok_or(AccountsdbPluginServiceError::InvalidPluginPath)?; .ok_or(GeyserPluginServiceError::InvalidPluginPath)?;
unsafe { unsafe {
let result = plugin_manager.load_plugin(libpath.to_str().unwrap(), config_file); let result = plugin_manager.load_plugin(libpath.to_str().unwrap(), config_file);
@ -193,7 +190,7 @@ impl AccountsDbPluginService {
"Failed to load the plugin library: {:?}, error: {:?}", "Failed to load the plugin library: {:?}, error: {:?}",
libpath, err libpath, err
); );
return Err(AccountsdbPluginServiceError::PluginLoadError(msg)); return Err(GeyserPluginServiceError::PluginLoadError(msg));
} }
} }
Ok(()) Ok(())

4
accountsdb-plugin-manager/src/lib.rs → geyser-plugin-manager/src/lib.rs
View File

@ -1,8 +1,8 @@
pub mod accounts_update_notifier; pub mod accounts_update_notifier;
pub mod accountsdb_plugin_manager;
pub mod accountsdb_plugin_service;
pub mod block_metadata_notifier; pub mod block_metadata_notifier;
pub mod block_metadata_notifier_interface; pub mod block_metadata_notifier_interface;
pub mod geyser_plugin_manager;
pub mod geyser_plugin_service;
pub mod slot_status_notifier; pub mod slot_status_notifier;
pub mod slot_status_observer; pub mod slot_status_observer;
pub mod transaction_notifier; pub mod transaction_notifier;

12
accountsdb-plugin-manager/src/slot_status_notifier.rs → geyser-plugin-manager/src/slot_status_notifier.rs
View File

@ -1,7 +1,7 @@
use { use {
crate::accountsdb_plugin_manager::AccountsDbPluginManager, crate::geyser_plugin_manager::GeyserPluginManager,
log::*, log::*,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::SlotStatus, solana_geyser_plugin_interface::geyser_plugin_interface::SlotStatus,
solana_measure::measure::Measure, solana_measure::measure::Measure,
solana_metrics::*, solana_metrics::*,
solana_sdk::clock::Slot, solana_sdk::clock::Slot,
@ -22,7 +22,7 @@ pub trait SlotStatusNotifierInterface {
pub type SlotStatusNotifier = Arc<RwLock<dyn SlotStatusNotifierInterface + Sync + Send>>; pub type SlotStatusNotifier = Arc<RwLock<dyn SlotStatusNotifierInterface + Sync + Send>>;
pub struct SlotStatusNotifierImpl { pub struct SlotStatusNotifierImpl {
plugin_manager: Arc<RwLock<AccountsDbPluginManager>>, plugin_manager: Arc<RwLock<GeyserPluginManager>>,
} }
impl SlotStatusNotifierInterface for SlotStatusNotifierImpl { impl SlotStatusNotifierInterface for SlotStatusNotifierImpl {
@ -40,7 +40,7 @@ impl SlotStatusNotifierInterface for SlotStatusNotifierImpl {
} }
impl SlotStatusNotifierImpl { impl SlotStatusNotifierImpl {
pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self { pub fn new(plugin_manager: Arc<RwLock<GeyserPluginManager>>) -> Self {
Self { plugin_manager } Self { plugin_manager }
} }
@ -51,7 +51,7 @@ impl SlotStatusNotifierImpl {
} }
for plugin in plugin_manager.plugins.iter_mut() { for plugin in plugin_manager.plugins.iter_mut() {
let mut measure = Measure::start("accountsdb-plugin-update-slot"); let mut measure = Measure::start("geyser-plugin-update-slot");
match plugin.update_slot_status(slot, parent, slot_status) { match plugin.update_slot_status(slot, parent, slot_status) {
Err(err) => { Err(err) => {
error!( error!(
@ -71,7 +71,7 @@ impl SlotStatusNotifierImpl {
} }
measure.stop(); measure.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-update-slot-us", "geyser-plugin-update-slot-us",
measure.as_us() as usize, measure.as_us() as usize,
1000, 1000,
1000 1000

0
accountsdb-plugin-manager/src/slot_status_observer.rs → geyser-plugin-manager/src/slot_status_observer.rs
View File

14
accountsdb-plugin-manager/src/transaction_notifier.rs → geyser-plugin-manager/src/transaction_notifier.rs
View File

@ -1,8 +1,8 @@
/// Module responsible for notifying plugins of transactions /// Module responsible for notifying plugins of transactions
use { use {
crate::accountsdb_plugin_manager::AccountsDbPluginManager, crate::geyser_plugin_manager::GeyserPluginManager,
log::*, log::*,
solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{ solana_geyser_plugin_interface::geyser_plugin_interface::{
ReplicaTransactionInfo, ReplicaTransactionInfoVersions, ReplicaTransactionInfo, ReplicaTransactionInfoVersions,
}, },
solana_measure::measure::Measure, solana_measure::measure::Measure,
@ -16,9 +16,9 @@ use {
/// This implementation of TransactionNotifier is passed to the rpc's TransactionStatusService /// This implementation of TransactionNotifier is passed to the rpc's TransactionStatusService
/// at the validator startup. TransactionStatusService invokes the notify_transaction method /// at the validator startup. TransactionStatusService invokes the notify_transaction method
/// for new transactions. The implementation in turn invokes the notify_transaction of each /// for new transactions. The implementation in turn invokes the notify_transaction of each
/// plugin enabled with transaction notification managed by the AccountsDbPluginManager. /// plugin enabled with transaction notification managed by the GeyserPluginManager.
pub(crate) struct TransactionNotifierImpl { pub(crate) struct TransactionNotifierImpl {
plugin_manager: Arc<RwLock<AccountsDbPluginManager>>, plugin_manager: Arc<RwLock<GeyserPluginManager>>,
} }
impl TransactionNotifier for TransactionNotifierImpl { impl TransactionNotifier for TransactionNotifierImpl {
@ -29,7 +29,7 @@ impl TransactionNotifier for TransactionNotifierImpl {
transaction_status_meta: &TransactionStatusMeta, transaction_status_meta: &TransactionStatusMeta,
transaction: &SanitizedTransaction, transaction: &SanitizedTransaction,
) { ) {
let mut measure = Measure::start("accountsdb-plugin-notify_plugins_of_transaction_info"); let mut measure = Measure::start("geyser-plugin-notify_plugins_of_transaction_info");
let transaction_log_info = let transaction_log_info =
Self::build_replica_transaction_info(signature, transaction_status_meta, transaction); Self::build_replica_transaction_info(signature, transaction_status_meta, transaction);
@ -64,7 +64,7 @@ impl TransactionNotifier for TransactionNotifierImpl {
} }
measure.stop(); measure.stop();
inc_new_counter_debug!( inc_new_counter_debug!(
"accountsdb-plugin-notify_plugins_of_transaction_info-us", "geyser-plugin-notify_plugins_of_transaction_info-us",
measure.as_us() as usize, measure.as_us() as usize,
10000, 10000,
10000 10000
@ -73,7 +73,7 @@ impl TransactionNotifier for TransactionNotifierImpl {
} }
impl TransactionNotifierImpl { impl TransactionNotifierImpl {
pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self { pub fn new(plugin_manager: Arc<RwLock<GeyserPluginManager>>) -> Self {
Self { plugin_manager } Self { plugin_manager }
} }

2
local-cluster/src/validator_configs.rs
View File

@ -15,7 +15,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
account_shrink_paths: config.account_shrink_paths.clone(), account_shrink_paths: config.account_shrink_paths.clone(),
rpc_config: config.rpc_config.clone(), rpc_config: config.rpc_config.clone(),
accountsdb_repl_service_config: config.accountsdb_repl_service_config.clone(), accountsdb_repl_service_config: config.accountsdb_repl_service_config.clone(),
accountsdb_plugin_config_files: config.accountsdb_plugin_config_files.clone(), geyser_plugin_config_files: config.geyser_plugin_config_files.clone(),
rpc_addrs: config.rpc_addrs, rpc_addrs: config.rpc_addrs,
pubsub_config: config.pubsub_config.clone(), pubsub_config: config.pubsub_config.clone(),
snapshot_config: config.snapshot_config.clone(), snapshot_config: config.snapshot_config.clone(),

4
runtime/src/accounts_db.rs
View File

@ -415,7 +415,7 @@ pub enum LoadedAccountAccessor<'a> {
Cached(Option<Cow<'a, CachedAccount>>), Cached(Option<Cow<'a, CachedAccount>>),
} }
mod accountsdb_plugin_utils; mod geyser_plugin_utils;
impl<'a> LoadedAccountAccessor<'a> { impl<'a> LoadedAccountAccessor<'a> {
fn check_and_get_loaded_account(&mut self) -> LoadedAccount { fn check_and_get_loaded_account(&mut self) -> LoadedAccount {
@ -1072,7 +1072,7 @@ pub struct AccountsDb {
/// for incremental snapshot support. /// for incremental snapshot support.
zero_lamport_accounts_to_purge_after_full_snapshot: DashSet<(Slot, Pubkey)>, zero_lamport_accounts_to_purge_after_full_snapshot: DashSet<(Slot, Pubkey)>,
/// AccountsDbPlugin accounts update notifier /// GeyserPlugin accounts update notifier
accounts_update_notifier: Option<AccountsUpdateNotifier>, accounts_update_notifier: Option<AccountsUpdateNotifier>,
filler_account_count: usize, filler_account_count: usize,

28
runtime/src/accounts_db/accountsdb_plugin_utils.rs → runtime/src/accounts_db/geyser_plugin_utils.rs
View File

@ -10,7 +10,7 @@ use {
}; };
#[derive(Default)] #[derive(Default)]
pub struct AccountsDbPluginNotifyAtSnapshotRestoreStats { pub struct GeyserPluginNotifyAtSnapshotRestoreStats {
pub total_accounts: usize, pub total_accounts: usize,
pub skipped_accounts: usize, pub skipped_accounts: usize,
pub notified_accounts: usize, pub notified_accounts: usize,
@ -20,7 +20,7 @@ pub struct AccountsDbPluginNotifyAtSnapshotRestoreStats {
pub elapsed_notifying_us: usize, pub elapsed_notifying_us: usize,
} }
impl AccountsDbPluginNotifyAtSnapshotRestoreStats { impl GeyserPluginNotifyAtSnapshotRestoreStats {
pub fn report(&self) { pub fn report(&self) {
datapoint_info!( datapoint_info!(
"accountsdb_plugin_notify_account_restore_from_snapshot_summary", "accountsdb_plugin_notify_account_restore_from_snapshot_summary",
@ -46,7 +46,7 @@ impl AccountsDb {
let mut slots = self.storage.all_slots(); let mut slots = self.storage.all_slots();
let mut notified_accounts: HashSet<Pubkey> = HashSet::default(); let mut notified_accounts: HashSet<Pubkey> = HashSet::default();
let mut notify_stats = AccountsDbPluginNotifyAtSnapshotRestoreStats::default(); let mut notify_stats = GeyserPluginNotifyAtSnapshotRestoreStats::default();
slots.sort_by(|a, b| b.cmp(a)); slots.sort_by(|a, b| b.cmp(a));
for slot in slots { for slot in slots {
@ -75,7 +75,7 @@ impl AccountsDb {
&self, &self,
slot: Slot, slot: Slot,
notified_accounts: &mut HashSet<Pubkey>, notified_accounts: &mut HashSet<Pubkey>,
notify_stats: &mut AccountsDbPluginNotifyAtSnapshotRestoreStats, notify_stats: &mut GeyserPluginNotifyAtSnapshotRestoreStats,
) { ) {
let slot_stores = self.storage.get_slot_stores(slot).unwrap(); let slot_stores = self.storage.get_slot_stores(slot).unwrap();
@ -117,7 +117,7 @@ impl AccountsDb {
slot: Slot, slot: Slot,
notified_accounts: &mut HashSet<Pubkey>, notified_accounts: &mut HashSet<Pubkey>,
accounts_to_stream: &HashMap<Pubkey, StoredAccountMeta>, accounts_to_stream: &HashMap<Pubkey, StoredAccountMeta>,
notify_stats: &mut AccountsDbPluginNotifyAtSnapshotRestoreStats, notify_stats: &mut GeyserPluginNotifyAtSnapshotRestoreStats,
) { ) {
let notifier = self let notifier = self
.accounts_update_notifier .accounts_update_notifier
@ -168,18 +168,18 @@ pub mod tests {
}; };
impl AccountsDb { impl AccountsDb {
pub fn set_accountsdb_plugin_notifer(&mut self, notifier: Option<AccountsUpdateNotifier>) { pub fn set_geyser_plugin_notifer(&mut self, notifier: Option<AccountsUpdateNotifier>) {
self.accounts_update_notifier = notifier; self.accounts_update_notifier = notifier;
} }
} }
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct AccountsDbTestPlugin { struct GeyserTestPlugin {
pub accounts_notified: DashMap<Pubkey, Vec<(Slot, AccountSharedData)>>, pub accounts_notified: DashMap<Pubkey, Vec<(Slot, AccountSharedData)>>,
pub is_startup_done: AtomicBool, pub is_startup_done: AtomicBool,
} }
impl AccountsUpdateNotifierInterface for AccountsDbTestPlugin { impl AccountsUpdateNotifierInterface for GeyserTestPlugin {
/// Notified when an account is updated at runtime, due to transaction activities /// Notified when an account is updated at runtime, due to transaction activities
fn notify_account_update( fn notify_account_update(
&self, &self,
@ -221,7 +221,7 @@ pub mod tests {
account1_lamports = 2; account1_lamports = 2;
let account1 = AccountSharedData::new(account1_lamports, 1, account1.owner()); let account1 = AccountSharedData::new(account1_lamports, 1, account1.owner());
accounts.store_uncached(slot0, &[(&key1, &account1)]); accounts.store_uncached(slot0, &[(&key1, &account1)]);
let notifier = AccountsDbTestPlugin::default(); let notifier = GeyserTestPlugin::default();
let key2 = solana_sdk::pubkey::new_rand(); let key2 = solana_sdk::pubkey::new_rand();
let account2_lamports: u64 = 100; let account2_lamports: u64 = 100;
@ -231,7 +231,7 @@ pub mod tests {
accounts.store_uncached(slot0, &[(&key2, &account2)]); accounts.store_uncached(slot0, &[(&key2, &account2)]);
let notifier = Arc::new(RwLock::new(notifier)); let notifier = Arc::new(RwLock::new(notifier));
accounts.set_accountsdb_plugin_notifer(Some(notifier.clone())); accounts.set_geyser_plugin_notifer(Some(notifier.clone()));
accounts.notify_account_restore_from_snapshot(); accounts.notify_account_restore_from_snapshot();
@ -279,7 +279,7 @@ pub mod tests {
let slot1 = 1; let slot1 = 1;
let account1 = AccountSharedData::new(account1_lamports, 1, account1.owner()); let account1 = AccountSharedData::new(account1_lamports, 1, account1.owner());
accounts.store_uncached(slot1, &[(&key1, &account1)]); accounts.store_uncached(slot1, &[(&key1, &account1)]);
let notifier = AccountsDbTestPlugin::default(); let notifier = GeyserTestPlugin::default();
let key3 = solana_sdk::pubkey::new_rand(); let key3 = solana_sdk::pubkey::new_rand();
let account3_lamports: u64 = 300; let account3_lamports: u64 = 300;
@ -288,7 +288,7 @@ pub mod tests {
accounts.store_uncached(slot1, &[(&key3, &account3)]); accounts.store_uncached(slot1, &[(&key3, &account3)]);
let notifier = Arc::new(RwLock::new(notifier)); let notifier = Arc::new(RwLock::new(notifier));
accounts.set_accountsdb_plugin_notifer(Some(notifier.clone())); accounts.set_geyser_plugin_notifer(Some(notifier.clone()));
accounts.notify_account_restore_from_snapshot(); accounts.notify_account_restore_from_snapshot();
@ -324,10 +324,10 @@ pub mod tests {
fn test_notify_account_at_accounts_update() { fn test_notify_account_at_accounts_update() {
let mut accounts = AccountsDb::new_single_for_tests_with_caching(); let mut accounts = AccountsDb::new_single_for_tests_with_caching();
let notifier = AccountsDbTestPlugin::default(); let notifier = GeyserTestPlugin::default();
let notifier = Arc::new(RwLock::new(notifier)); let notifier = Arc::new(RwLock::new(notifier));
accounts.set_accountsdb_plugin_notifer(Some(notifier.clone())); accounts.set_geyser_plugin_notifer(Some(notifier.clone()));
// Account with key1 is updated twice in two different slots -- should only get notified twice. // Account with key1 is updated twice in two different slots -- should only get notified twice.
// Account with key2 is updated slot0, should get notified once // Account with key2 is updated slot0, should get notified once

6
test-validator/src/lib.rs
View File

@ -109,7 +109,7 @@ pub struct TestValidatorGenesis {
pub authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>, pub authorized_voter_keypairs: Arc<RwLock<Vec<Arc<Keypair>>>>,
pub max_ledger_shreds: Option<u64>, pub max_ledger_shreds: Option<u64>,
pub max_genesis_archive_unpacked_size: Option<u64>, pub max_genesis_archive_unpacked_size: Option<u64>,
pub accountsdb_plugin_config_files: Option<Vec<PathBuf>>, pub geyser_plugin_config_files: Option<Vec<PathBuf>>,
pub accounts_db_caching_enabled: bool, pub accounts_db_caching_enabled: bool,
deactivate_feature_set: HashSet<Pubkey>, deactivate_feature_set: HashSet<Pubkey>,
} }
@ -136,7 +136,7 @@ impl Default for TestValidatorGenesis {
authorized_voter_keypairs: Arc::<RwLock<Vec<Arc<Keypair>>>>::default(), authorized_voter_keypairs: Arc::<RwLock<Vec<Arc<Keypair>>>>::default(),
max_ledger_shreds: Option::<u64>::default(), max_ledger_shreds: Option::<u64>::default(),
max_genesis_archive_unpacked_size: Option::<u64>::default(), max_genesis_archive_unpacked_size: Option::<u64>::default(),
accountsdb_plugin_config_files: Option::<Vec<PathBuf>>::default(), geyser_plugin_config_files: Option::<Vec<PathBuf>>::default(),
accounts_db_caching_enabled: bool::default(), accounts_db_caching_enabled: bool::default(),
deactivate_feature_set: HashSet::<Pubkey>::default(), deactivate_feature_set: HashSet::<Pubkey>::default(),
} }
@ -669,7 +669,7 @@ impl TestValidator {
}); });
let mut validator_config = ValidatorConfig { let mut validator_config = ValidatorConfig {
accountsdb_plugin_config_files: config.accountsdb_plugin_config_files.clone(), geyser_plugin_config_files: config.geyser_plugin_config_files.clone(),
accounts_db_caching_enabled: config.accounts_db_caching_enabled, accounts_db_caching_enabled: config.accounts_db_caching_enabled,
rpc_addrs: Some(( rpc_addrs: Some((
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()), SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), node.info.rpc.port()),

13
validator/src/bin/solana-test-validator.rs
View File

@ -312,12 +312,13 @@ fn main() {
), ),
) )
.arg( .arg(
Arg::with_name("accountsdb_plugin_config") Arg::with_name("geyser_plugin_config")
.long("accountsdb-plugin-config") .long("geyser-plugin-config")
.alias("accountsdb-plugin-config")
.value_name("FILE") .value_name("FILE")
.takes_value(true) .takes_value(true)
.multiple(true) .multiple(true)
.help("Specify the configuration file for the AccountsDb plugin."), .help("Specify the configuration file for the Geyser plugin."),
) )
.arg( .arg(
Arg::with_name("no_accounts_db_caching") Arg::with_name("no_accounts_db_caching")
@ -683,9 +684,9 @@ fn main() {
genesis.bind_ip_addr(bind_address); genesis.bind_ip_addr(bind_address);
} }
if matches.is_present("accountsdb_plugin_config") { if matches.is_present("geyser_plugin_config") {
genesis.accountsdb_plugin_config_files = Some( genesis.geyser_plugin_config_files = Some(
values_t_or_exit!(matches, "accountsdb_plugin_config", String) values_t_or_exit!(matches, "geyser_plugin_config", String)
.into_iter() .into_iter()
.map(PathBuf::from) .map(PathBuf::from)
.collect(), .collect(),

13
validator/src/main.rs
View File

@ -1370,12 +1370,13 @@ pub fn main() {
.help("Number of threads to use for servicing AccountsDb Replication requests"), .help("Number of threads to use for servicing AccountsDb Replication requests"),
) )
.arg( .arg(
Arg::with_name("accountsdb_plugin_config") Arg::with_name("geyser_plugin_config")
.long("accountsdb-plugin-config") .long("geyser-plugin-config")
.alias("accountsdb-plugin-config")
.value_name("FILE") .value_name("FILE")
.takes_value(true) .takes_value(true)
.multiple(true) .multiple(true)
.help("Specify the configuration file for the AccountsDb plugin."), .help("Specify the configuration file for the Geyser plugin."),
) )
.arg( .arg(
Arg::with_name("halt_on_known_validators_accounts_hash_mismatch") Arg::with_name("halt_on_known_validators_accounts_hash_mismatch")
@ -2240,9 +2241,9 @@ pub fn main() {
None None
}; };
let accountsdb_plugin_config_files = if matches.is_present("accountsdb_plugin_config") { let geyser_plugin_config_files = if matches.is_present("geyser_plugin_config") {
Some( Some(
values_t_or_exit!(matches, "accountsdb_plugin_config", String) values_t_or_exit!(matches, "geyser_plugin_config", String)
.into_iter() .into_iter()
.map(PathBuf::from) .map(PathBuf::from)
.collect(), .collect(),
@ -2297,7 +2298,7 @@ pub fn main() {
rpc_scan_and_fix_roots: matches.is_present("rpc_scan_and_fix_roots"), rpc_scan_and_fix_roots: matches.is_present("rpc_scan_and_fix_roots"),
}, },
accountsdb_repl_service_config, accountsdb_repl_service_config,
accountsdb_plugin_config_files, geyser_plugin_config_files,
rpc_addrs: value_t!(matches, "rpc_port", u16).ok().map(|rpc_port| { rpc_addrs: value_t!(matches, "rpc_port", u16).ok().map(|rpc_port| {
( (
SocketAddr::new(rpc_bind_address, rpc_port), SocketAddr::new(rpc_bind_address, rpc_port),