Add ability to interact with a Bigtable with a custom instance id (backport #23779) (#24325)

* Refactor validator bigtable config

(cherry picked from commit 63ee00e647)

* bigtable: add a config ctor for `LedgerStorage`

(cherry picked from commit f513195468)

* bigtable: allow custom instance names

(cherry picked from commit 9b32b72990)

# Conflicts:
#	validator/Cargo.toml

* Add ability to query bigtable via solana-test-validator, with hidden params

(cherry picked from commit 9c60991cd3)

* Fix conflicts

Co-authored-by: Tyera Eulberg <tyera@solana.com>
Co-authored-by: Trent Nelson <trent@solana.com>

validator/Cargo.toml

@@ -48,6 +48,7 @@ solana-rpc = { path = "../rpc", version = "=1.10.9" }
 solana-runtime = { path = "../runtime", version = "=1.10.9" }
 solana-sdk = { path = "../sdk", version = "=1.10.9" }
 solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.9" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.10.9" }
 solana-streamer = { path = "../streamer", version = "=1.10.9" }
 solana-test-validator = { path = "../test-validator", version = "=1.10.9" }
 solana-version = { path = "../version", version = "=1.10.9" }

validator/src/bin/solana-test-validator.rs

@@ -12,7 +12,10 @@ use {
     solana_client::rpc_client::RpcClient,
     solana_core::tower_storage::FileTowerStorage,
     solana_faucet::faucet::{run_local_faucet_with_port, FAUCET_PORT},
-    solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
+    solana_rpc::{
+        rpc::{JsonRpcConfig, RpcBigtableConfig},
+        rpc_pubsub_service::PubSubConfig,
+    },
     solana_sdk::{
         account::AccountSharedData,
         clock::Slot,
@@ -155,6 +158,23 @@ fn main() {
                 .validator(solana_validator::port_validator)
                 .help("Enable JSON RPC on this port, and the next port for the RPC websocket"),
         )
+        .arg(
+            Arg::with_name("enable_rpc_bigtable_ledger_storage")
+                .long("enable-rpc-bigtable-ledger-storage")
+                .takes_value(false)
+                .hidden(true)
+                .help("Fetch historical transaction info from a BigTable instance \
+                       as a fallback to local ledger data"),
+        )
+        .arg(
+            Arg::with_name("rpc_bigtable_instance")
+                .long("rpc-bigtable-instance")
+                .value_name("INSTANCE_NAME")
+                .takes_value(true)
+                .hidden(true)
+                .default_value("solana-ledger")
+                .help("Name of BigTable instance to target"),
+        )
         .arg(
             Arg::with_name("rpc_pubsub_enable_vote_subscription")
                 .long("rpc-pubsub-enable-vote-subscription")
@@ -618,6 +638,16 @@ fn main() {
         None
     };
 
+    let rpc_bigtable_config = if matches.is_present("enable_rpc_bigtable_ledger_storage") {
+        Some(RpcBigtableConfig {
+            enable_bigtable_ledger_upload: false,
+            bigtable_instance_name: value_t_or_exit!(matches, "rpc_bigtable_instance", String),
+            timeout: None,
+        })
+    } else {
+        None
+    };
+
     genesis
         .ledger_path(&ledger_path)
         .tower_storage(tower_storage)
@@ -628,6 +658,7 @@ fn main() {
         .rpc_config(JsonRpcConfig {
             enable_rpc_transaction_history: true,
             enable_cpi_and_log_storage: true,
+            rpc_bigtable_config,
             faucet_addr,
             ..JsonRpcConfig::default_for_test()
         })

validator/src/main.rs

@@ -37,7 +37,10 @@ use {
     solana_perf::recycler::enable_recycler_warming,
     solana_poh::poh_service,
     solana_replica_lib::accountsdb_repl_server::AccountsDbReplServiceConfig,
-    solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
+    solana_rpc::{
+        rpc::{JsonRpcConfig, RpcBigtableConfig},
+        rpc_pubsub_service::PubSubConfig,
+    },
     solana_runtime::{
         accounts_db::{
             AccountShrinkThreshold, AccountsDbConfig, DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE,
@@ -1200,6 +1203,14 @@ pub fn main() {
                 .default_value("30")
                 .help("Number of seconds before timing out RPC requests backed by BigTable"),
         )
+        .arg(
+            Arg::with_name("rpc_bigtable_instance_name")
+                .long("rpc-bigtable-instance-name")
+                .takes_value(true)
+                .value_name("INSTANCE_NAME")
+                .default_value(solana_storage_bigtable::DEFAULT_INSTANCE_NAME)
+                .help("Name of the Bigtable instance to upload to")
+        )
         .arg(
             Arg::with_name("rpc_pubsub_worker_threads")
                 .long("rpc-pubsub-worker-threads")
@@ -2268,6 +2279,20 @@ pub fn main() {
         warn!("--minimal-rpc-api is now the default behavior. This flag is deprecated and can be removed from the launch args")
     }
 
+    let rpc_bigtable_config = if matches.is_present("enable_rpc_bigtable_ledger_storage")
+        || matches.is_present("enable_bigtable_ledger_upload")
+    {
+        Some(RpcBigtableConfig {
+            enable_bigtable_ledger_upload: matches.is_present("enable_bigtable_ledger_upload"),
+            bigtable_instance_name: value_t_or_exit!(matches, "rpc_bigtable_instance_name", String),
+            timeout: value_t!(matches, "rpc_bigtable_timeout", u64)
+                .ok()
+                .map(Duration::from_secs),
+        })
+    } else {
+        None
+    };
+
     let mut validator_config = ValidatorConfig {
         require_tower: matches.is_present("require_tower"),
         tower_storage,
@@ -2283,9 +2308,7 @@ pub fn main() {
         rpc_config: JsonRpcConfig {
             enable_rpc_transaction_history: matches.is_present("enable_rpc_transaction_history"),
             enable_cpi_and_log_storage: matches.is_present("enable_cpi_and_log_storage"),
-            enable_bigtable_ledger_storage: matches
-                .is_present("enable_rpc_bigtable_ledger_storage"),
-            enable_bigtable_ledger_upload: matches.is_present("enable_bigtable_ledger_upload"),
+            rpc_bigtable_config,
             faucet_addr: matches.value_of("rpc_faucet_addr").map(|address| {
                 solana_net_utils::parse_host_port(address).expect("failed to parse faucet address")
             }),
@@ -2303,9 +2326,6 @@ pub fn main() {
             ),
             rpc_threads: value_t_or_exit!(matches, "rpc_threads", usize),
             rpc_niceness_adj: value_t_or_exit!(matches, "rpc_niceness_adj", i8),
-            rpc_bigtable_timeout: value_t!(matches, "rpc_bigtable_timeout", u64)
-                .ok()
-                .map(Duration::from_secs),
             account_indexes: account_indexes.clone(),
             rpc_scan_and_fix_roots: matches.is_present("rpc_scan_and_fix_roots"),
         },