Accountsdb stream plugin improvement (#20419) (#20573)

* Accountsdb stream plugin improvement (#20419)

Support using connection pooling and use multiple threads to do Postgres db operations. The performance is improved from 1500 RPS to 40,000 RPS measured during validator start.

Support multiple plugins at the same time.

* Fixed a fmt issue
This commit is contained in:
Lijun Wang
2021-10-10 15:24:12 -07:00
committed by GitHub
parent e5dc8d731b
commit 50cb612ae1
11 changed files with 563 additions and 143 deletions

View File

@@ -1716,6 +1716,7 @@ pub fn main() {
.long("accountsdb-plugin-config")
.value_name("FILE")
.takes_value(true)
.multiple(true)
.hidden(true)
.help("Specify the configuration file for the AccountsDb plugin."),
)
@@ -2277,9 +2278,16 @@ pub fn main() {
.ok()
.or_else(|| get_cluster_shred_version(&entrypoint_addrs));
let accountsdb_plugin_config_file = matches
.value_of("accountsdb_plugin_config")
.map(PathBuf::from);
let accountsdb_plugin_config_files = if matches.is_present("accountsdb_plugin_config") {
Some(
values_t_or_exit!(matches, "accountsdb_plugin_config", String)
.into_iter()
.map(PathBuf::from)
.collect(),
)
} else {
None
};
let mut validator_config = ValidatorConfig {
require_tower: matches.is_present("require_tower"),
@@ -2322,7 +2330,7 @@ pub fn main() {
account_indexes: account_indexes.clone(),
rpc_scan_and_fix_roots: matches.is_present("rpc_scan_and_fix_roots"),
},
accountsdb_plugin_config_file,
accountsdb_plugin_config_files,
rpc_addrs: value_t!(matches, "rpc_port", u16).ok().map(|rpc_port| {
(
SocketAddr::new(rpc_bind_address, rpc_port),