From 817f2fe1adea8e36c37a354eb31b19eab77ae704 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Wed, 17 Feb 2021 02:56:24 +0000
Subject: [PATCH] Add --force arg for bigtable upload (#15361)

(cherry picked from commit 98e3e570d2c8932e8ddc174a3a34cccd0730e50d)

Co-authored-by: Tyera Eulberg <tyera@solana.com>
---
 core/src/bigtable_upload_service.rs |  1 +
 ledger-tool/src/bigtable.rs         | 14 ++++++++++++++
 ledger/src/bigtable_upload.rs       |  5 ++++-
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/core/src/bigtable_upload_service.rs b/core/src/bigtable_upload_service.rs
index 23db772af0..08ee16e38b 100644
--- a/core/src/bigtable_upload_service.rs
+++ b/core/src/bigtable_upload_service.rs
@@ -74,6 +74,7 @@ impl BigTableUploadService {
                 start_slot,
                 Some(end_slot),
                 true,
+                false,
                 exit.clone(),
             ));
 
diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs
index 103f83a900..c26fd51451 100644
--- a/ledger-tool/src/bigtable.rs
+++ b/ledger-tool/src/bigtable.rs
@@ -20,6 +20,7 @@ async fn upload(
     starting_slot: Slot,
     ending_slot: Option<Slot>,
     allow_missing_metadata: bool,
+    force_reupload: bool,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None)
         .await
@@ -31,6 +32,7 @@ async fn upload(
         starting_slot,
         ending_slot,
         allow_missing_metadata,
+        force_reupload,
         Arc::new(AtomicBool::new(false)),
     )
     .await
@@ -247,6 +249,16 @@ impl BigTableSubCommand for App<'_, '_> {
                                 .long("allow-missing-metadata")
                                 .takes_value(false)
                                 .help("Don't panic if transaction metadata is missing"),
+                        )
+                        .arg(
+                            Arg::with_name("force_reupload")
+                                .long("force")
+                                .takes_value(false)
+                                .help(
+                                    "Force reupload of any blocks already present in BigTable instance\
+                                    Note: reupload will *not* delete any data from the tx-by-addr table;\
+                                    Use with care.",
+                                ),
                         ),
                 )
                 .subcommand(
@@ -389,6 +401,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
             let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
             let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
             let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
+            let force_reupload = arg_matches.is_present("force_reupload");
             let blockstore =
                 crate::open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary, None);
 
@@ -397,6 +410,7 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
                 starting_slot,
                 ending_slot,
                 allow_missing_metadata,
+                force_reupload,
             ))
         }
         ("first-available-block", Some(_arg_matches)) => runtime.block_on(first_available_block()),
diff --git a/ledger/src/bigtable_upload.rs b/ledger/src/bigtable_upload.rs
index dd43975531..7c07bee13f 100644
--- a/ledger/src/bigtable_upload.rs
+++ b/ledger/src/bigtable_upload.rs
@@ -25,6 +25,7 @@ pub async fn upload_confirmed_blocks(
     starting_slot: Slot,
     ending_slot: Option<Slot>,
     allow_missing_metadata: bool,
+    force_reupload: bool,
     exit: Arc<AtomicBool>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let mut measure = Measure::start("entire upload");
@@ -64,7 +65,7 @@ pub async fn upload_confirmed_blocks(
     );
 
     // Gather the blocks that are already present in bigtable, by slot
-    let bigtable_slots = {
+    let bigtable_slots = if !force_reupload {
         let mut bigtable_slots = vec![];
         let first_blockstore_slot = *blockstore_slots.first().unwrap();
         let last_blockstore_slot = *blockstore_slots.last().unwrap();
@@ -95,6 +96,8 @@ pub async fn upload_confirmed_blocks(
             .into_iter()
             .filter(|slot| *slot <= last_blockstore_slot)
             .collect::<Vec<_>>()
+    } else {
+        Vec::new()
     };
 
     // The blocks that still need to be uploaded is the difference between what's already in the