From b64fb295a15c870df330b0db7f96dffa15b79a15 Mon Sep 17 00:00:00 2001
From: Michael Vines <mvines@gmail.com>
Date: Fri, 4 Sep 2020 14:54:38 -0700
Subject: [PATCH] Add LARGEST_CONFIRMED_ROOT_UPLOAD_DELAY

---
 core/src/bigtable_upload_service.rs | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/core/src/bigtable_upload_service.rs b/core/src/bigtable_upload_service.rs
index b8a445504b..23db772af0 100644
--- a/core/src/bigtable_upload_service.rs
+++ b/core/src/bigtable_upload_service.rs
@@ -7,6 +7,14 @@ use std::{
 };
 use tokio::runtime;
 
+// Delay uploading the largest confirmed root for this many slots.  This is done in an attempt to
+// ensure that the `CacheBlockTimeService` has had enough time to add the block time for the root
+// before it's uploaded to BigTable.
+//
+// A more direct connection between CacheBlockTimeService and BigTableUploadService would be
+// preferable...
+const LARGEST_CONFIRMED_ROOT_UPLOAD_DELAY: usize = 100;
+
 pub struct BigTableUploadService {
     thread: JoinHandle<()>,
 }
@@ -43,18 +51,19 @@ impl BigTableUploadService {
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         exit: Arc<AtomicBool>,
     ) {
-        let mut starting_slot = 0;
+        let mut start_slot = 0;
         loop {
             if exit.load(Ordering::Relaxed) {
                 break;
             }
 
-            let max_confirmed_root = block_commitment_cache
+            let end_slot = block_commitment_cache
                 .read()
                 .unwrap()
-                .highest_confirmed_root();
+                .highest_confirmed_root()
+                .saturating_sub(LARGEST_CONFIRMED_ROOT_UPLOAD_DELAY as u64);
 
-            if max_confirmed_root == starting_slot {
+            if end_slot <= start_slot {
                 std::thread::sleep(std::time::Duration::from_secs(1));
                 continue;
             }
@@ -62,14 +71,14 @@ impl BigTableUploadService {
             let result = runtime.block_on(solana_ledger::bigtable_upload::upload_confirmed_blocks(
                 blockstore.clone(),
                 bigtable_ledger_storage.clone(),
-                starting_slot,
-                Some(max_confirmed_root),
+                start_slot,
+                Some(end_slot),
                 true,
                 exit.clone(),
             ));
 
             match result {
-                Ok(()) => starting_slot = max_confirmed_root,
+                Ok(()) => start_slot = end_slot,
                 Err(err) => {
                     warn!("bigtable: upload_confirmed_blocks: {}", err);
                     std::thread::sleep(std::time::Duration::from_secs(2));