gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

AccountsDb plugin framework (#20047)

Browse Source 
Summary of Changes

Create a plugin mechanism in the accounts update path so that accounts data can be streamed out to external data stores (be it Kafka or Postgres). The plugin mechanism allows

Data stores of connection strings/credentials to be configured,
Accounts with patterns to be streamed
PostgreSQL implementation of the streaming for different destination stores to be plugged in.

The code comprises 4 major parts:

accountsdb-plugin-intf: defines the plugin interface which concrete plugin should implement.
accountsdb-plugin-manager: manages the load/unload of plugins and provide interfaces which the validator can notify of accounts update to plugins.
accountsdb-plugin-postgres: the concrete plugin implementation for PostgreSQL
The validator integrations: updated streamed right after snapshot restore and after account update from transaction processing or other real updates.
The plugin is optionally loaded on demand by new validator CLI argument -- there is no impact if the plugin is not loaded.
This commit is contained in:
Lijun Wang
2021-09-30 14:26:17 -07:00
committed by GitHub
parent 9ef1f06b3f
commit fe97cb2ddf
41 changed files with 1512 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
runtime/src/accounts.rs
View File

@ -5,6 +5,7 @@ use crate::{
ACCOUNTS_DB_CONFIG_FOR_TESTING,
},
accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
accounts_update_notifier_interface::AccountsUpdateNotifier,
ancestors::Ancestors,
bank::{
NonceRollbackFull, NonceRollbackInfo, RentDebits, TransactionCheckResult,
@ -139,6 +140,7 @@ impl Accounts {
caching_enabled,
shrink_ratio,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
}
@ -156,6 +158,7 @@ impl Accounts {
caching_enabled,
shrink_ratio,
Some(ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS),
None,
)
}
@ -166,6 +169,7 @@ impl Accounts {
caching_enabled: bool,
shrink_ratio: AccountShrinkThreshold,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Self {
Self {
accounts_db: Arc::new(AccountsDb::new_with_config(
@ -175,6 +179,7 @@ impl Accounts {
caching_enabled,
shrink_ratio,
accounts_db_config,
accounts_update_notifier,
)),
account_locks: Mutex::new(AccountLocks::default()),
}

39
runtime/src/accounts_db.rs
View File

@ -28,6 +28,7 @@ use crate::{
SlotSlice, ZeroLamport, ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS,
ACCOUNTS_INDEX_CONFIG_FOR_TESTING,
},
accounts_update_notifier_interface::AccountsUpdateNotifier,
ancestors::Ancestors,
append_vec::{AppendVec, StoredAccountMeta, StoredMeta, StoredMetaWriteVersion},
cache_hash_data::CacheHashData,
@ -1037,6 +1038,9 @@ pub struct AccountsDb {
/// Zero-lamport accounts that are *not* purged during clean because they need to stay alive
/// for incremental snapshot support.
zero_lamport_accounts_to_purge_after_full_snapshot: DashSet<(Slot, Pubkey)>,
/// AccountsDbPlugin accounts update notifier
accounts_update_notifier: Option<AccountsUpdateNotifier>,
}
#[derive(Debug, Default)]
@ -1504,6 +1508,7 @@ impl AccountsDb {
shrink_ratio: AccountShrinkThreshold::default(),
dirty_stores: DashMap::default(),
zero_lamport_accounts_to_purge_after_full_snapshot: DashSet::default(),
accounts_update_notifier: None,
}
}
@ -1515,6 +1520,7 @@ impl AccountsDb {
false,
AccountShrinkThreshold::default(),
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
}
@ -1525,6 +1531,7 @@ impl AccountsDb {
caching_enabled: bool,
shrink_ratio: AccountShrinkThreshold,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Self {
let accounts_index =
AccountsIndex::new(accounts_db_config.as_ref().and_then(|x| x.index.clone()));
@ -1539,6 +1546,7 @@ impl AccountsDb {
account_indexes,
caching_enabled,
shrink_ratio,
accounts_update_notifier,
..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path)
}
} else {
@ -4205,6 +4213,7 @@ impl AccountsDb {
{
let stored_size = offsets[1] - offsets[0];
storage.add_account(stored_size);
infos.push(AccountInfo {
store_id: storage.append_vec_id(),
offset: offsets[0],
@ -4224,6 +4233,7 @@ impl AccountsDb {
self.stats
.store_find_store
.fetch_add(total_storage_find_us, Ordering::Relaxed);
infos
}
@ -5921,6 +5931,16 @@ impl AccountsDb {
pub fn store_cached(&self, slot: Slot, accounts: &[(&Pubkey, &AccountSharedData)]) {
self.store(slot, accounts, self.caching_enabled);
if let Some(accounts_update_notifier) = &self.accounts_update_notifier {
let notifier = &accounts_update_notifier.read().unwrap();
for account in accounts {
let pubkey = account.0;
let account = account.1;
notifier.notify_account_update(slot, pubkey, account);
}
}
}
/// Store the account update.
@ -6662,6 +6682,24 @@ impl AccountsDb {
}
}
}
pub fn notify_account_restore_from_snapshot(&self) {
if let Some(accounts_update_notifier) = &self.accounts_update_notifier {
let notifier = &accounts_update_notifier.read().unwrap();
let slots = self.storage.all_slots();
for slot in &slots {
let slot_stores = self.storage.get_slot_stores(*slot).unwrap();
let slot_stores = slot_stores.read().unwrap();
for (_, storage_entry) in slot_stores.iter() {
let accounts = storage_entry.all_accounts();
for account in &accounts {
notifier.notify_account_restore_from_snapshot(*slot, account);
}
}
}
}
}
}
#[cfg(test)]
@ -6684,6 +6722,7 @@ impl AccountsDb {
caching_enabled,
shrink_ratio,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
}

25
runtime/src/accounts_update_notifier_interface.rs Normal file
View File

@ -0,0 +1,25 @@
use {
crate::append_vec::StoredAccountMeta,
solana_sdk::{account::AccountSharedData, clock::Slot, pubkey::Pubkey},
std::sync::{Arc, RwLock},
};
pub trait AccountsUpdateNotifierInterface: std::fmt::Debug {
/// Notified when an account is updated at runtime, due to transaction activities
fn notify_account_update(&self, slot: Slot, pubkey: &Pubkey, account: &AccountSharedData);
/// Notified when the AccountsDb is initialized at start when restored
/// from a snapshot.
fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta);
/// Notified when a slot is optimistically confirmed
fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>);
/// Notified when a slot is marked frozen.
fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>);
/// Notified when a slot is rooted.
fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>);
}
pub type AccountsUpdateNotifier = Arc<RwLock<dyn AccountsUpdateNotifierInterface + Sync + Send>>;

5
runtime/src/bank.rs
View File

@ -40,6 +40,7 @@ use crate::{
ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING,
},
accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
accounts_update_notifier_interface::AccountsUpdateNotifier,
ancestors::{Ancestors, AncestorsForSerialization},
blockhash_queue::BlockhashQueue,
builtins::{self, ActivationType, Builtin, Builtins},
@ -1138,6 +1139,7 @@ impl Bank {
shrink_ratio,
debug_do_not_add_builtins,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
}
@ -1163,6 +1165,7 @@ impl Bank {
shrink_ratio,
debug_do_not_add_builtins,
Some(ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS),
None,
)
}
@ -1178,6 +1181,7 @@ impl Bank {
shrink_ratio: AccountShrinkThreshold,
debug_do_not_add_builtins: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Self {
let accounts = Accounts::new_with_config(
paths,
@ -1186,6 +1190,7 @@ impl Bank {
accounts_db_caching_enabled,
shrink_ratio,
accounts_db_config,
accounts_update_notifier,
);
let mut bank = Self::default_with_accounts(accounts);
bank.ancestors = Ancestors::from(vec![bank.slot()]);

1
runtime/src/lib.rs
View File

@ -7,6 +7,7 @@ pub mod accounts_db;
pub mod accounts_hash;
pub mod accounts_index;
pub mod accounts_index_storage;
pub mod accounts_update_notifier_interface;
pub mod ancestors;
pub mod append_vec;
pub mod bank;

12
runtime/src/serde_snapshot.rs
View File

@ -6,6 +6,7 @@ use {
BankHashInfo,
},
accounts_index::AccountSecondaryIndexes,
accounts_update_notifier_interface::AccountsUpdateNotifier,
ancestors::Ancestors,
append_vec::{AppendVec, StoredMetaWriteVersion},
bank::{Bank, BankFieldsToDeserialize, BankRc},
@ -204,6 +205,7 @@ pub(crate) fn bank_from_streams<R>(
shrink_ratio: AccountShrinkThreshold,
verify_index: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> std::result::Result<Bank, Error>
where
R: Read,
@ -242,6 +244,7 @@ where
shrink_ratio,
verify_index,
accounts_db_config,
accounts_update_notifier,
)?;
Ok(bank)
}};
@ -335,6 +338,7 @@ fn reconstruct_bank_from_fields<E>(
shrink_ratio: AccountShrinkThreshold,
verify_index: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Result<Bank, Error>
where
E: SerializableStorage + std::marker::Sync,
@ -350,6 +354,7 @@ where
shrink_ratio,
verify_index,
accounts_db_config,
accounts_update_notifier,
)?;
accounts_db.freeze_accounts(
&Ancestors::from(&bank_fields.ancestors),
@ -404,6 +409,7 @@ fn reconstruct_accountsdb_from_fields<E>(
shrink_ratio: AccountShrinkThreshold,
verify_index: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Result<AccountsDb, Error>
where
E: SerializableStorage + std::marker::Sync,
@ -415,6 +421,7 @@ where
caching_enabled,
shrink_ratio,
accounts_db_config,
accounts_update_notifier,
);
let AccountsDbFields(
@ -530,6 +537,10 @@ where
.fetch_add(snapshot_version, Ordering::Relaxed);
accounts_db.generate_index(limit_load_slot_count_from_snapshot, verify_index);
let mut measure_notify = Measure::start("accounts_notify");
accounts_db.notify_account_restore_from_snapshot();
measure_notify.stop();
datapoint_info!(
"reconstruct_accountsdb_from_fields()",
("remap-time-us", measure_remap.as_us(), i64),
@ -538,6 +549,7 @@ where
num_collisions.load(Ordering::Relaxed),
i64
),
("accountsdb-notify-at-start-us", measure_notify.as_us(), i64),
);
Ok(accounts_db)

2
runtime/src/serde_snapshot/tests.rs
View File

@ -84,6 +84,7 @@ where
AccountShrinkThreshold::default(),
false,
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
}
@ -246,6 +247,7 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
AccountShrinkThreshold::default(),
false,
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
dbank.src = ref_sc;

13
runtime/src/snapshot_utils.rs
View File

@ -2,6 +2,7 @@ use {
crate::{
accounts_db::{AccountShrinkThreshold, AccountsDbConfig},
accounts_index::AccountSecondaryIndexes,
accounts_update_notifier_interface::AccountsUpdateNotifier,
bank::{Bank, BankSlotDelta},
builtins::Builtins,
hardened_unpack::{unpack_snapshot, ParallelSelector, UnpackError, UnpackedAppendVecMap},
@ -734,6 +735,7 @@ pub fn bank_from_snapshot_archives(
accounts_db_skip_shrink: bool,
verify_index: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Result<(Bank, BankFromArchiveTimings)> {
check_are_snapshots_compatible(
full_snapshot_archive_info,
@ -798,6 +800,7 @@ pub fn bank_from_snapshot_archives(
shrink_ratio,
verify_index,
accounts_db_config,
accounts_update_notifier,
)?;
measure_rebuild.stop();
info!("{}", measure_rebuild);
@ -844,6 +847,7 @@ pub fn bank_from_latest_snapshot_archives(
accounts_db_skip_shrink: bool,
verify_index: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Result<(
Bank,
BankFromArchiveTimings,
@ -887,6 +891,7 @@ pub fn bank_from_latest_snapshot_archives(
accounts_db_skip_shrink,
verify_index,
accounts_db_config,
accounts_update_notifier,
)?;
verify_bank_against_expected_slot_hash(
@ -1424,6 +1429,7 @@ fn rebuild_bank_from_snapshots(
shrink_ratio: AccountShrinkThreshold,
verify_index: bool,
accounts_db_config: Option<AccountsDbConfig>,
accounts_update_notifier: Option<AccountsUpdateNotifier>,
) -> Result<Bank> {
let (full_snapshot_version, full_snapshot_root_paths) =
verify_unpacked_snapshots_dir_and_version(
@ -1472,6 +1478,7 @@ fn rebuild_bank_from_snapshots(
shrink_ratio,
verify_index,
accounts_db_config,
accounts_update_notifier,
),
}?,
)
@ -2645,6 +2652,7 @@ mod tests {
false,
false,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
@ -2736,6 +2744,7 @@ mod tests {
false,
false,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
@ -2846,6 +2855,7 @@ mod tests {
false,
false,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
@ -2945,6 +2955,7 @@ mod tests {
false,
false,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
@ -3086,6 +3097,7 @@ mod tests {
false,
false,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
assert_eq!(
@ -3148,6 +3160,7 @@ mod tests {
false,
false,
Some(ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
assert_eq!(