gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add ability to interact with a Bigtable with a custom instance id (backport #23779) (#24325)

Browse Source 
* Refactor validator bigtable config

(cherry picked from commit 63ee00e647)

* bigtable: add a config ctor for `LedgerStorage`

(cherry picked from commit f513195468)

* bigtable: allow custom instance names

(cherry picked from commit 9b32b72990)

# Conflicts:
#	validator/Cargo.toml

* Add ability to query bigtable via solana-test-validator, with hidden params

(cherry picked from commit 9c60991cd3)

* Fix conflicts

Co-authored-by: Tyera Eulberg <tyera@solana.com>
Co-authored-by: Trent Nelson <trent@solana.com>
This commit is contained in:
mergify[bot]
2022-04-13 19:00:47 -06:00
committed by GitHub
parent 32dea4427b
commit ec1d06240c
8 changed files with 277 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
ledger-tool/src/bigtable.rs
View File

@@ -34,8 +34,9 @@ async fn upload(
starting_slot: Slot,
ending_slot: Option<Slot>,
force_reupload: bool,
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config)
.await
.map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
@@ -50,17 +51,22 @@ async fn upload(
.await
}
async fn delete_slots(slots: Vec<Slot>, dry_run: bool) -> Result<(), Box<dyn std::error::Error>> {
let read_only = dry_run;
let bigtable = solana_storage_bigtable::LedgerStorage::new(read_only, None, None)
async fn delete_slots(
slots: Vec<Slot>,
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let dry_run = config.read_only;
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config)
.await
.map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
solana_ledger::bigtable_delete::delete_confirmed_blocks(bigtable, slots, dry_run).await
}
async fn first_available_block() -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(true, None, None).await?;
async fn first_available_block(
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config).await?;
match bigtable.get_first_available_block().await? {
Some(block) => println!("{}", block),
None => println!("No blocks available"),
@@ -69,8 +75,12 @@ async fn first_available_block() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
async fn block(slot: Slot, output_format: OutputFormat) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
async fn block(
slot: Slot,
output_format: OutputFormat,
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config)
.await
.map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
@@ -101,8 +111,12 @@ async fn block(slot: Slot, output_format: OutputFormat) -> Result<(), Box<dyn st
Ok(())
}
async fn blocks(starting_slot: Slot, limit: usize) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
async fn blocks(
starting_slot: Slot,
limit: usize,
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config)
.await
.map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
@@ -116,11 +130,10 @@ async fn blocks(starting_slot: Slot, limit: usize) -> Result<(), Box<dyn std::er
async fn compare_blocks(
starting_slot: Slot,
limit: usize,
credential_path: String,
config: solana_storage_bigtable::LedgerStorageConfig,
ref_config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
assert!(!credential_path.is_empty());
let owned_bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
let owned_bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config)
.await
.map_err(|err| format!("failed to connect to owned bigtable: {:?}", err))?;
let owned_bigtable_slots = owned_bigtable
@@ -130,10 +143,9 @@ async fn compare_blocks(
"owned bigtable {} blocks found ",
owned_bigtable_slots.len()
);
let reference_bigtable =
solana_storage_bigtable::LedgerStorage::new(false, None, Some(credential_path))
.await
.map_err(|err| format!("failed to connect to reference bigtable: {:?}", err))?;
let reference_bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(ref_config)
.await
.map_err(|err| format!("failed to connect to reference bigtable: {:?}", err))?;
let reference_bigtable_slots = reference_bigtable
.get_confirmed_blocks(starting_slot, limit)
@@ -160,8 +172,9 @@ async fn confirm(
signature: &Signature,
verbose: bool,
output_format: OutputFormat,
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config)
.await
.map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
@@ -211,8 +224,9 @@ pub async fn transaction_history(
verbose: bool,
show_transactions: bool,
query_chunk_size: usize,
config: solana_storage_bigtable::LedgerStorageConfig,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(true, None, None).await?;
let bigtable = solana_storage_bigtable::LedgerStorage::new_with_config(config).await?;
let mut loaded_block: Option<(Slot, ConfirmedBlock)> = None;
while limit > 0 {
@@ -308,6 +322,15 @@ impl BigTableSubCommand for App<'_, '_> {
.about("Ledger data on a BigTable instance")
.setting(AppSettings::InferSubcommands)
.setting(AppSettings::SubcommandRequiredElseHelp)
.arg(
Arg::with_name("rpc_bigtable_instance_name")
.global(true)
.long("rpc-bigtable-instance-name")
.takes_value(true)
.value_name("INSTANCE_NAME")
.default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.help("Name of the target Bigtable instance")
)
.subcommand(
SubCommand::with_name("upload")
.about("Upload the ledger to BigTable")
@@ -417,7 +440,8 @@ impl BigTableSubCommand for App<'_, '_> {
.required(true)
.default_value("1000")
.help("Maximum number of slots to check"),
).arg(
)
.arg(
Arg::with_name("reference_credential")
.long("reference-credential")
.short("c")
@@ -425,6 +449,14 @@ impl BigTableSubCommand for App<'_, '_> {
.takes_value(true)
.required(true)
.help("File path for a credential to a reference bigtable"),
)
.arg(
Arg::with_name("reference_instance_name")
.long("reference-instance-name")
.takes_value(true)
.value_name("INSTANCE_NAME")
.default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.help("Name of the reference Bigtable instance to compare to")
),
)
.subcommand(
@@ -521,7 +553,28 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
let verbose = matches.is_present("verbose");
let output_format = OutputFormat::from_matches(matches, "output_format", verbose);
let future = match matches.subcommand() {
// this is kinda stupid, but there seems to be a bug in clap when a subcommand
// arg is marked both `global(true)` and `default_value("default_value")`.
// despite the "global", when the arg is specified on the subcommand, its value
// is not propagated down to the (sub)subcommand args, resulting in the default
// value when queried there. similarly, if the arg is specified on the
// (sub)subcommand, the value is not propagated back up to the subcommand args,
// again resulting in the default value. the arg having declared a
// `default_value()` obviates `is_present(...)` tests since they will always
// return true. so we consede and compare against the expected default. :/
let (subcommand, sub_matches) = matches.subcommand();
let on_command = matches
.value_of("rpc_bigtable_instance_name")
.map(|v| v != solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
.unwrap_or(false);
let instance_name = if on_command {
value_t_or_exit!(matches, "rpc_bigtable_instance_name", String)
} else {
let sub_matches = sub_matches.as_ref().unwrap();
value_t_or_exit!(sub_matches, "rpc_bigtable_instance_name", String)
};
let future = match (subcommand, sub_matches) {
("upload", Some(arg_matches)) => {
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
@@ -531,41 +584,79 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
AccessType::TryPrimaryThenSecondary,
None,
);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(upload(
blockstore,
starting_slot,
ending_slot,
force_reupload,
config,
))
}
("delete-slots", Some(arg_matches)) => {
let slots = values_t_or_exit!(arg_matches, "slots", Slot);
let dry_run = !arg_matches.is_present("force");
runtime.block_on(delete_slots(slots, dry_run))
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: !arg_matches.is_present("force"),
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(delete_slots(slots, config))
}
("first-available-block", Some(_arg_matches)) => {
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: true,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(first_available_block(config))
}
("first-available-block", Some(_arg_matches)) => runtime.block_on(first_available_block()),
("block", Some(arg_matches)) => {
let slot = value_t_or_exit!(arg_matches, "slot", Slot);
runtime.block_on(block(slot, output_format))
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(block(slot, output_format, config))
}
("blocks", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let limit = value_t_or_exit!(arg_matches, "limit", usize);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(blocks(starting_slot, limit))
runtime.block_on(blocks(starting_slot, limit, config))
}
("compare-blocks", Some(arg_matches)) => {
let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
let limit = value_t_or_exit!(arg_matches, "limit", usize);
let reference_credential_filepath =
value_t_or_exit!(arg_matches, "reference_credential", String);
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
let credential_path = Some(value_t_or_exit!(
arg_matches,
"reference_credential",
String
));
let ref_instance_name =
value_t_or_exit!(arg_matches, "reference_instance_name", String);
let ref_config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
credential_path,
instance_name: ref_instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(compare_blocks(
starting_slot,
limit,
reference_credential_filepath,
))
runtime.block_on(compare_blocks(starting_slot, limit, config, ref_config))
}
("confirm", Some(arg_matches)) => {
let signature = arg_matches
@@ -573,8 +664,13 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
.unwrap()
.parse()
.expect("Invalid signature");
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: false,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(confirm(&signature, verbose, output_format))
runtime.block_on(confirm(&signature, verbose, output_format, config))
}
("transaction-history", Some(arg_matches)) => {
let address = pubkey_of(arg_matches, "address").unwrap();
@@ -587,6 +683,11 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
.value_of("until")
.map(|signature| signature.parse().expect("Invalid signature"));
let show_transactions = arg_matches.is_present("show_transactions");
let config = solana_storage_bigtable::LedgerStorageConfig {
read_only: true,
instance_name,
..solana_storage_bigtable::LedgerStorageConfig::default()
};
runtime.block_on(transaction_history(
&address,
@@ -596,6 +697,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
verbose,
show_transactions,
query_chunk_size,
config,
))
}
_ => unreachable!(),