You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2019/12/21 14:38:39 UTC

[lucene-solr] branch jira/SOLR-13101 updated: SOLR-13101: Concurrency tests for SHARED collection. (#1081)

This is an automated email from the ASF dual-hosted git repository.

yonik pushed a commit to branch jira/SOLR-13101
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/jira/SOLR-13101 by this push:
     new 036ce53  SOLR-13101: Concurrency tests for SHARED collection. (#1081)
036ce53 is described below

commit 036ce539525aa9c0eb0270d94e4105cd1a0313e9
Author: Bilal Waheed <23...@users.noreply.github.com>
AuthorDate: Sat Dec 21 06:38:26 2019 -0800

    SOLR-13101: Concurrency tests for SHARED collection. (#1081)
    
    * SOLR-13101: Concurrency tests for SHARED collection.
    
    -Concurrency tests for SHARED collection.
    -On core container shutdown, shutdown the background blob delete manager too.
    -Use correct casing for SharedCoreStage enum.
    -Added INDEXING_BATCH_FINISHED to indicate the end of a batch (BLOB_PUSH_FINISHED does not help if push itself runs into error)
    
    * -CR feedback.
---
 .../java/org/apache/solr/core/CoreContainer.java   |   1 +
 .../solr/store/blob/process/CorePullTask.java      |   4 +-
 .../apache/solr/store/blob/process/CorePusher.java |  10 +-
 .../solr/store/blob/util/BlobStoreUtils.java       |   4 +-
 .../shared/SharedCoreConcurrencyController.java    |  24 +-
 .../processor/DistributedZkUpdateProcessor.java    |  15 +-
 .../store/shared/SharedCoreConcurrencyTest.java    | 913 ++++++++++++++-------
 7 files changed, 630 insertions(+), 341 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 56c8354..db05a55 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1071,6 +1071,7 @@ public class CoreContainer {
         
         if (sharedStoreManager != null) {
           sharedStoreManager.getBlobProcessManager().shutdown();
+          sharedStoreManager.getBlobDeleteManager().shutdown();
         }
       }
 
diff --git a/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java b/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java
index f100a4d..44eccca 100644
--- a/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java
+++ b/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java
@@ -243,7 +243,7 @@ public class CorePullTask implements DeduplicatingList.Deduplicatable<String> {
         return;
       }
 
-      concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BlobPullStarted);
+      concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BLOB_PULL_STARTED);
       // Get blob metadata
       String blobCoreMetadataName = BlobStoreUtils.buildBlobStoreMetadataName(shardVersionMetadata.getMetadataSuffix());
       blobMetadata = blobClient.pullCoreMetadata(pullCoreInfo.getSharedStoreName(), blobCoreMetadataName);
@@ -351,7 +351,7 @@ public class CorePullTask implements DeduplicatingList.Deduplicatable<String> {
       }
     }
     this.callback.finishedPull(this, blobMetadata, syncStatus, message);
-    concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BlobPullFinished);
+    concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BLOB_PULL_FINISHED);
   }
 
   void finishedPull(BlobCoreMetadata blobCoreMetadata, CoreSyncStatus syncStatus, String message) throws InterruptedException {
diff --git a/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java b/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java
index 2712b8d..2f6e215 100644
--- a/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java
+++ b/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java
@@ -85,7 +85,7 @@ public class CorePusher {
           throw new SolrException(ErrorCode.SERVER_ERROR, "Can't find core " + coreName);
         }
         try {
-          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPushStarted);
+          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PUSH_STARTED);
 
           IndexCommit latestCommit = core.getDeletionPolicy().getLatestCommit();
           if (latestCommit == null) {
@@ -130,7 +130,7 @@ public class CorePusher {
           // begin the push process 
           CorePushPull pushPull = new CorePushPull(blobClient, deleteManager, pushPullData, resolutionResult, localCoreMetadata, coreVersionMetadata.getBlobCoreMetadata());
           BlobCoreMetadata blobCoreMetadata = pushPull.pushToBlobStore(coreVersionMetadata.getMetadataSuffix(), newMetadataSuffix);
-          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPushed);
+          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PUSHED);
           // at this point we've pushed the new metadata file with the newMetadataSuffix and now need to write to zookeeper
           SharedShardMetadataController shardSharedMetadataController = coreContainer.getSharedStoreManager().getSharedShardMetadataController();
           SharedShardVersionMetadata newShardVersionMetadata = null;
@@ -154,17 +154,17 @@ public class CorePusher {
             }
             throw ex;
           }
-          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.ZkUpdateFinished);
+          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.ZK_UPDATE_FINISHED);
 
           assert newMetadataSuffix.equals(newShardVersionMetadata.getMetadataSuffix());
           // after successful update to zookeeper, update core version metadata with new version info
           // and we can also give soft guarantee that core is up to date w.r.to shared store, until unless failures happen and leadership changes 
           concurrencyController.updateCoreVersionMetadata(collectionName, shardName, coreName, newShardVersionMetadata, blobCoreMetadata, /* softGuaranteeOfEquality */ true);
-          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LocalCacheUpdateFinished);
+          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LOCAL_CACHE_UPDATE_FINISHED);
           log.info(String.format("Successfully pushed to shared store, pushLockTime=%s pushPullData=%s", lockAcquisitionTime, pushPullData.toString()));
         } finally {
           try {
-            concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPushFinished);
+            concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PUSH_FINISHED);
             if (snapshotDirPath != null) {
               // we are done with push we can now remove the snapshot directory
               removeSnapshotDirectory(core, snapshotDirPath);
diff --git a/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java b/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java
index a9e26e5..c31ea79 100644
--- a/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java
+++ b/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java
@@ -83,7 +83,7 @@ public class BlobStoreUtils {
           log.info("sync successful, nothing to pull, collection=" + collectionName + " shard=" + shardName + " coreName=" + coreName);
           return;
         }
-        concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPullStarted);
+        concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PULL_STARTED);
         try {
           // Get blob metadata
           String blobCoreMetadataName = BlobStoreUtils.buildBlobStoreMetadataName(shardVersionMetadata.getMetadataSuffix());
@@ -128,7 +128,7 @@ public class BlobStoreUtils {
                 collectionName, shardName, coreName));
           }
         } finally {
-          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPullFinished);
+          concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PULL_FINISHED);
         }
       } catch (Exception ex) {
         // wrap every thrown exception in a solr exception
diff --git a/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java b/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java
index 2adde86..06c8714 100644
--- a/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java
+++ b/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java
@@ -390,43 +390,47 @@ public class SharedCoreConcurrencyController {
     /**
      * Necessary locks have been acquired and we have started to pull from the shared store.
      */
-    BlobPullStarted,
+    BLOB_PULL_STARTED,
     /**
      * Pull(either successful or failed) has ended and we are about to release the necessary locks.
      */
-    BlobPullFinished,
+    BLOB_PULL_FINISHED,
     /**
      * We have received an indexing batch but necessary locks have not been acquired yet.
      */
-    IndexingBatchReceived,
+    INDEXING_BATCH_RECEIVED,
     /**
      * We are passed the shared store pull stage(if applicable) and are in sync with shared store.
      * Now we will proceed with local indexing.
      */
-    LocalIndexingStarted,
+    LOCAL_INDEXING_STARTED,
     /**
      * Local indexing finished but not have been pushed to shared store.
      */
-    LocalIndexingFinished,
+    LOCAL_INDEXING_FINISHED,
     /**
      * Necessary locks have been acquired and push to shared store has started.
      */
-    BlobPushStarted,
+    BLOB_PUSH_STARTED,
     /**
      * Files have been pushed to blob.
      */
-    BlobPushed,
+    BLOB_PUSHED,
     /**
      * Zookeeper has been successfully updated with new metadata.
      */
-    ZkUpdateFinished,
+    ZK_UPDATE_FINISHED,
     /**
      * Local cache {@link #coresVersionMetadata} has been successfully updated with new metadata.
      */
-    LocalCacheUpdateFinished,
+    LOCAL_CACHE_UPDATE_FINISHED,
     /**
      * Push(either successful or failed) has ended and we are about to release the necessary locks.
      */
-    BlobPushFinished
+    BLOB_PUSH_FINISHED,
+    /**
+     * Indexing batch(either successful or failed) has ended and we are about to release the necessary locks.
+     */
+    INDEXING_BATCH_FINISHED
   }
 }
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index 5a8abdb..f06845b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -1067,6 +1067,14 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
   @Override
   protected void doClose() {
     if (corePullLock != null) {
+      try {
+        SharedCoreConcurrencyController concurrencyController =
+            req.getCore().getCoreContainer().getSharedStoreManager().getSharedCoreConcurrencyController();
+        concurrencyController.recordState(cloudDesc.getCollectionName(), cloudDesc.getShardId(), req.getCore().getName(),
+            SharedCoreStage.INDEXING_BATCH_FINISHED);
+      } catch (Exception ex) {
+        SolrException.log(log, "Error recording the finish of SHARED core indexing batch", ex);
+      }
       // release read lock
       corePullLock.readLock().unlock();
     }
@@ -1097,12 +1105,13 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
   }
 
 
+  // TODO: Refactor most of shared replica logic out into a separate class.
   private void writeToSharedStore() {
     String collectionName = cloudDesc.getCollectionName();
     String shardName = cloudDesc.getShardId();
     String coreName = req.getCore().getName();
     SharedCoreConcurrencyController concurrencyController = req.getCore().getCoreContainer().getSharedStoreManager().getSharedCoreConcurrencyController();
-    concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LocalIndexingFinished);
+    concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LOCAL_INDEXING_FINISHED);
 
     log.info("Attempting to initiate index update write to shared store for collection=" + collectionName +
         " and shard=" + shardName + " using core=" + coreName);
@@ -1134,7 +1143,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
 
     CoreContainer coreContainer = req.getCore().getCoreContainer();
     SharedCoreConcurrencyController concurrencyController = coreContainer.getSharedStoreManager().getSharedCoreConcurrencyController();
-    concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.IndexingBatchReceived);
+    concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.INDEXING_BATCH_RECEIVED);
     corePullLock = concurrencyController.getCorePullLock(collectionName, shardName, coreName);
     // from this point on wards we should always exit this method with read lock (no matter failure or what)
     try {
@@ -1155,7 +1164,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
       // acquire lock for the whole duration of update
       // we should always leave with read lock acquired(failure or success), since it is the job of close method to release it
       corePullLock.readLock().lock();
-      concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LocalIndexingStarted);
+      concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LOCAL_INDEXING_STARTED);
     }
   }
 
diff --git a/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java b/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java
index 8ec1fad..7753d9f 100644
--- a/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java
+++ b/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java
@@ -16,31 +16,40 @@
  */
 package org.apache.solr.store.shared;
 
+import java.io.File;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
 import org.apache.commons.io.FileUtils;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrCore;
 import org.apache.solr.store.blob.client.CoreStorageClient;
 import org.apache.solr.store.shared.SharedCoreConcurrencyController.SharedCoreStage;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -49,411 +58,677 @@ import org.slf4j.LoggerFactory;
 /**
  * Tests around synchronization of concurrent indexing, pushes and pulls
  * happening on a core of a shared collection {@link DocCollection#getSharedIndex()}
- * todo: add tests for failover scenarios and involve query pulls 
  */
 public class SharedCoreConcurrencyTest extends SolrCloudSharedStoreTestCase {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+  private static final String COLLECTION_NAME = "sharedCollection";
+  private static final String SHARD_NAME = "shard1";
+  /**
+   * Number of serial indexing iterations for each test. This is the main setting, queries and failover iterations
+   * stop after indexing ends. Higher the value, longer the tests would run.
+   */
+  private static int INDEXING_ITERATIONS = TEST_NIGHTLY ? 100 : 20;
+  /**
+   * Maximum number of concurrent indexing requests per indexing iteration.
+   */
+  private static int MAX_NUM_OF_CONCURRENT_INDEXING_REQUESTS_PER_ITERATION = 10;
+  /**
+   * Maximum number of docs per indexing request.
+   */
+  private static int MAX_NUM_OF_DOCS_PER_INDEXING_REQUEST = 100;
+  /**
+   * Indexing can fail because of leader failures (especially when test {@link #includeFailovers()}).
+   * The test will re-attempt up till this number of times before bailing out. For test to succeed,
+   * indexing request have to succeed in these many attempts.
+   */
+  private static int MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST = 10;
+  /**
+   * Maximum number of concurrent query requests per query iteration.
+   */
+  private static int MAX_NUM_OF_CONCURRENT_QUERY_REQUESTS_PER_ITERATION = 10;
+  /**
+   * Querying is faster than indexing, to pace it better with indexing, a delay is added between each query iteration.
+   */
+  private static int DELAY_MS_BETWEEN_EACH_QUERY_ITERATION = 50;
+  /**
+   * Minimum time between each failover.
+   */
+  private static int DELAY_MS_BETWEEN_EACH_FAILOVER_ITERATION = 500;
+  /**
+   * Path for local shared store
+   */
   private static Path sharedStoreRootPath;
 
+  /**
+   * Manages test state from start to end.
+   */
+  private TestState testState;
+
   @BeforeClass
   public static void setupClass() throws Exception {
     sharedStoreRootPath = createTempDir("tempDir");
   }
 
+  @Before
+  public void setupTest() throws Exception {
+    int numNodes = 4;
+    setupCluster(numNodes);
+    testState = new TestState();
+    setupSolrNodesForTest();
+
+    int maxShardsPerNode = 1;
+    // One less than number of nodes.
+    // The extra node will be used at the end of test to verify 
+    // the contents of shared store by querying for all docs on a new replica.
+    int numReplicas = numNodes - 1;
+    // Later on we can consider choosing random number of shards and replicas.
+    // To handle multiple shards, we need to update code where SHARD_NAME is used.
+    setupSharedCollectionWithShardNames(COLLECTION_NAME, maxShardsPerNode, numReplicas, SHARD_NAME);
+  }
+
+  @After
+  public void teardownTest() throws Exception {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+    FileUtils.cleanDirectory(sharedStoreRootPath.toFile());
+  }
+
   /**
-   * Tests issuing random number of concurrent indexing requests in a given range for a shared core and making sure they all succeed.
+   * Tests that concurrent indexing succeed.
    */
   @Test
-  public void testHighConcurrentIndexing() throws Exception {
-    int maxIndexingThreads = 100;
-    int maxDocsPerThread = 100;
-    testConcurrentIndexing(maxIndexingThreads, maxDocsPerThread);
+  public void testIndexing() throws Exception {
+    final boolean includeDeletes = false;
+    includeIndexing(includeDeletes);
+    run();
   }
 
   /**
-   * Test ensuring two indexing requests interleave in desired ways and succeed.
+   * Tests that concurrent indexing with concurrent queries succeed.
    */
-  //  @Test todo: leaking threads, test only issue
-  public void TODOtestPossibleInterleaving() throws Exception {
-
-    // completely serialized
-    testConcurrentIndexing(
-        new IndexingThreadInterleaver(
-            SharedCoreStage.BlobPushFinished,
-            SharedCoreStage.LocalIndexingStarted,
-            true));
-
-    // second pushes for first one too
-    testConcurrentIndexing(
-        new IndexingThreadInterleaver(
-            SharedCoreStage.LocalIndexingFinished, 
-            SharedCoreStage.BlobPushFinished,
-            true));
+  @Test
+  public void testIndexingQueries() throws Exception {
+    final boolean includeDeletes = false;
+    includeIndexing(includeDeletes);
+    includeQueries();
+    run();
   }
 
   /**
-   * Test ensuring two indexing requests do not interleave in impossible ways even when forced and still succeed.
+   * Tests that concurrent indexing with deletes and concurrent queries succeed.
    */
-  //  @Test todo: leaking threads, test only issue
-  public void TODOtestImpossibleInterleaving() throws Exception {
-    // push critical section
-    testConcurrentIndexing(
-        new IndexingThreadInterleaver(
-            SharedCoreStage.ZkUpdateFinished,
-            SharedCoreStage.ZkUpdateFinished,
-            false));
-
-    // another push critical section
-    testConcurrentIndexing(
-        new IndexingThreadInterleaver(
-            SharedCoreStage.ZkUpdateFinished,
-            SharedCoreStage.LocalCacheUpdateFinished,
-            false));
+  @Test
+  public void testIndexingQueriesDeletes() throws Exception {
+    final boolean includeDeletes = true;
+    includeIndexing(includeDeletes);
+    includeQueries();
+    run();
   }
 
-  private void testConcurrentIndexing(int maxIndexingThreads, int maxDocsPerThread) throws Exception {
-    int numIndexingThreads = new Random().nextInt(maxIndexingThreads) + 1;;
-    testConcurrentIndexing(numIndexingThreads, maxDocsPerThread, null);
+  /**
+   * Tests that concurrent indexing with deletes, concurrent queries and explicit failovers succeed.
+   */
+  // @Test 
+  // TODO: This test flaps time to time. The symptom of the failure is missing docs i.e. indexing is declared successful
+  //       but query could not reproduce all of the docs. I was able to repro this with NRT collection on vanilla 8.3 too. 
+  //       I have not root caused it yet. Keeping this test disabled, until the problem is root caused and fixed. 
+  public void todoTestIndexingQueriesDeletesFailovers() throws Exception { 
+    final boolean includeDeletes = true;
+    includeIndexing(includeDeletes);
+    includeQueries();
+    includeFailovers();
+    run();
   }
 
-  private void testConcurrentIndexing(IndexingThreadInterleaver interleaver) throws Exception {
-    testConcurrentIndexing(2, 10, interleaver);
+  /**
+   * It starts all the threads that are included in the test (indexing, queries and failovers) in parallel.
+   * Then wait for them to finish (run length depends on {@link #INDEXING_ITERATIONS}).
+   * At the end it makes sures that no critical section was breached and no unexpected error occurred.
+   * Then verify the contents of shared store by querying for all docs on a new replica.
+   */
+  private void run() throws Exception {
+    testState.startIncludedThreads();
+    testState.waitForThreadsToStop();
+    analyzeCoreConcurrencyStagesForBreaches();
+    testState.checkErrors();
+    Replica newReplica = addReplica();
+    queryNewReplicaAndVerifyAllDocsFound(newReplica);
   }
 
   /**
-   * Start desired number of concurrent indexing threads with each indexing random number of docs between 1 and maxDocsPerThread.
+   * Adds a thread to test, that goes over {@link #INDEXING_ITERATIONS} or until it is interrupted.
+   * In each iteration it creates between 1 and {@link #MAX_NUM_OF_CONCURRENT_INDEXING_REQUESTS_PER_ITERATION} threads
+   * by calling {@link #createIndexingThreads(int, int, boolean)}, starts them concurrently and wait for them to finish
+   * before going to next iteration. Each indexing thread adds between 1 and  {@link #MAX_NUM_OF_DOCS_PER_INDEXING_REQUEST}
+   * docs.
    *
-   * At the end it verifies everything got indexed successfully on leader. No critical section got breached.
-   * Also verify the integrity of shared store contents by pulling them on a follower replica.
+   * @param includeDeletes whether to randomly mark some docs for deletion and delete them in subsequent indexing requests
+   *                       or not
    */
-  private void testConcurrentIndexing(int numIndexingThreads, int maxDocsPerThread, IndexingThreadInterleaver interleaver) throws Exception {
-    setupCluster(2);
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-
-    // this map tracks the async pull queues per solr process
-    Map<String, Map<String, CountDownLatch>> solrProcessesTaskTracker = new HashMap<>();
-
-    JettySolrRunner solrProcess1 = cluster.getJettySolrRunner(0);
-    CoreStorageClient storageClient1 = setupLocalBlobStoreClient(sharedStoreRootPath, DEFAULT_BLOB_DIR_NAME);
-    setupTestSharedClientForNode(getBlobStorageProviderTestInstance(storageClient1), solrProcess1);
-    Map<String, CountDownLatch> asyncPullLatches1 = configureTestBlobProcessForNode(solrProcess1);
-
-    JettySolrRunner solrProcess2 = cluster.getJettySolrRunner(1);
-    CoreStorageClient storageClient2 = setupLocalBlobStoreClient(sharedStoreRootPath, DEFAULT_BLOB_DIR_NAME);
-    setupTestSharedClientForNode(getBlobStorageProviderTestInstance(storageClient2), solrProcess2);
-    Map<String, CountDownLatch> asyncPullLatches2 = configureTestBlobProcessForNode(solrProcess2);
-
-    solrProcessesTaskTracker.put(solrProcess1.getNodeName(), asyncPullLatches1);
-    solrProcessesTaskTracker.put(solrProcess2.getNodeName(), asyncPullLatches2);
-
-    String collectionName = "sharedCollection";
-    int maxShardsPerNode = 1;
-    int numReplicas = 2;
-    // specify a comma-delimited string of shard names for multiple shards when using
-    // an implicit router
-    String shardNames = "shard1";
-    setupSharedCollectionWithShardNames(collectionName, maxShardsPerNode, numReplicas, shardNames);
-
-    // get the leader replica and follower replicas
-    DocCollection collection = cloudClient.getZkStateReader().getClusterState().getCollection(collectionName);
-    Replica shardLeaderReplica = collection.getLeader("shard1");
-    Replica followerReplica = null;
-    for (Replica repl : collection.getSlice("shard1").getReplicas()) {
-      if (repl.getName() != shardLeaderReplica.getName()) {
-        followerReplica = repl;
-        break;
+  private void includeIndexing(boolean includeDeletes) {
+    Thread t = new Thread(() -> {
+      try {
+        for (int i = 0; i < INDEXING_ITERATIONS && !testState.stopRunning.get(); i++) {
+          int numIndexingThreads = random().nextInt(MAX_NUM_OF_CONCURRENT_INDEXING_REQUESTS_PER_ITERATION) + 1;
+          int numDocsToAddPerThread = random().nextInt(MAX_NUM_OF_DOCS_PER_INDEXING_REQUEST) + 1;
+          Thread[] indexingThreads = createIndexingThreads(numIndexingThreads, numDocsToAddPerThread, includeDeletes);
+          for (int j = 0; j < numIndexingThreads; j++) {
+            indexingThreads[j].start();
+          }
+          for (int j = 0; j < numIndexingThreads; j++) {
+            indexingThreads[j].join();
+          }
+          if (Thread.interrupted()) {
+            // we have been interrupted so we will stop running
+            testState.stopRunning.set(true);
+          }
+        }
+      } catch (Exception ex) {
+        testState.indexingErrors.add(ex.getMessage());
       }
-    }
+      // everything else stops running when indexing finishes
+      testState.stopRunning.set(true);
+    });
+    testState.includeThread(t);
+  }
 
-    
-   JettySolrRunner shardLeaderSolrRunner = null;
-   for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
-     if(solrRunner.getNodeName().equals(shardLeaderReplica.getNodeName())){
-       shardLeaderSolrRunner = solrRunner;
-       break;
-     }
-   }
-   List<String> progress = new ArrayList<>();
-
-   if(interleaver != null) {
-     configureTestSharedConcurrencyControllerForNode(shardLeaderSolrRunner, progress, interleaver);
-   } else {
-     configureTestSharedConcurrencyControllerForNode(shardLeaderSolrRunner, progress);
-   }
-
-    AtomicInteger totalDocs= new AtomicInteger(0);
+  /**
+   * Creates {@code numIndexingThreads} threads with each adding {@code numDocsToAddPerThread}.
+   *
+   * @param includeDeletes whether to randomly mark some docs for deletion and delete them in subsequent indexing requests
+   *                       or not
+   */
+  private Thread[] createIndexingThreads(int numIndexingThreads, int numDocsToAddPerThread, boolean includeDeletes) throws Exception {
     log.info("numIndexingThreads=" + numIndexingThreads);
     Thread[] indexingThreads = new Thread[numIndexingThreads];
-    ConcurrentLinkedQueue<String> indexingErrors = new ConcurrentLinkedQueue<>();
-    for (int i = 0; i < numIndexingThreads; i++) {
+    for (int i = 0; i < numIndexingThreads && !testState.stopRunning.get(); i++) {
       indexingThreads[i] = new Thread(() -> {
-        try {
-          // index between 1 to maxDocsPerThread docs
-          int numDocs = new Random().nextInt(maxDocsPerThread) + 1;
-          log.info("numDocs=" + numDocs);
-          UpdateRequest updateReq = new UpdateRequest();
-          for (int j = 0; j < numDocs; j++) {
-            int docId = totalDocs.incrementAndGet();
-            updateReq.add("id", Integer.toString(docId));
+        List<String> idsToAdd = new ArrayList<>();
+        // prepare the list of docs to add and delete outside the reattempt loop
+        for (int j = 0; j < numDocsToAddPerThread; j++) {
+          String docId = Integer.toString(testState.docIdGenerator.incrementAndGet());
+          idsToAdd.add(docId);
+        }
+        List<String> idsToDelete = testState.idBatchesToDelete.poll();
+
+        // attempt until succeeded or max attempts 
+        for (int j = 0; j < MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST; j++) {
+          try {
+            String message = "attempt=" + (j + 1) + " numDocsToAdd=" + numDocsToAddPerThread + " docsToAdd=" + idsToAdd.toString();
+            if (idsToDelete != null) {
+              message += " numDocsToDelete=" + idsToDelete.size() + " docsToDelete=" + idsToDelete.toString();
+            }
+            log.info(message);
+
+            UpdateRequest updateReq = new UpdateRequest();
+            for (int k = 0; k < idsToAdd.size(); k++) {
+              updateReq.add("id", idsToAdd.get(k));
+            }
+            if (includeDeletes && idsToDelete != null) {
+              updateReq.deleteById(idsToDelete);
+            }
+            processUpdateRequest(updateReq);
+
+            testState.numDocsIndexed.addAndGet(numDocsToAddPerThread);
+            if (idsToDelete != null) {
+              testState.idsDeleted.addAll(idsToDelete);
+            }
+
+            // randomly select some docs that can be deleted
+            if (includeDeletes) {
+              List<String> idsThatCanBeDeleted = new ArrayList<>();
+              for (String indexedId : idsToAdd) {
+                if (random().nextBoolean()) {
+                  idsThatCanBeDeleted.add(indexedId);
+                }
+              }
+              if (!idsThatCanBeDeleted.isEmpty()) {
+                testState.idBatchesToDelete.offer(idsThatCanBeDeleted);
+              }
+            }
+            // indexing was successful, stop attempting
+            break;
+          } catch (Exception ex) {
+            // last attempt also failed, record the error
+            if (j == MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST - 1) {
+              testState.indexingErrors.add(Throwables.getStackTraceAsString(ex));
+            }
           }
-          updateReq.commit(cloudClient, collectionName);
-        } catch (Exception ex) {
-          indexingErrors.add(ex.getMessage());
         }
       });
     }
+    return indexingThreads;
+  }
 
-    for (int i = 0; i < numIndexingThreads; i++) {
-      indexingThreads[i].start();
-    }
+  /**
+   * Sends update request to the server, randomly choosing whether to send it with commit=true or not
+   * Shared replica does not need an explicit commit since it always does an implicit hard commit but
+   * still it is valid to send an update with or without a commit, therefore, testing both.
+   */
+  private void processUpdateRequest(UpdateRequest request) throws Exception {
+    UpdateResponse response = random().nextBoolean()
+        ? request.process(cluster.getSolrClient(), COLLECTION_NAME)
+        : request.commit(cluster.getSolrClient(), COLLECTION_NAME);
 
-    for (int i = 0; i < numIndexingThreads; i++) {
-      indexingThreads[i].join();
+    if (response.getStatus() != 0) {
+      throw new RuntimeException("Update request failed with status=" + response.getStatus());
     }
+  }
 
-    log.info("totalDocs=" + totalDocs.intValue());
-
-    assertTrue(indexingErrors.toString(), indexingErrors.isEmpty());
+  /**
+   * Adds a thread to test, that goes over iterations until the test is stopped {@link TestState#stopRunning}.
+   * In each iteration it creates between 1 and {@link #MAX_NUM_OF_CONCURRENT_QUERY_REQUESTS_PER_ITERATION} threads
+   * by calling {@link #createQueryThreads(int)}, starts them concurrently and wait for them to finish
+   * before going to next iteration. To pace it better with indexing, {@link #DELAY_MS_BETWEEN_EACH_QUERY_ITERATION}
+   * delay is added between each query iteration.
+   */
+  private void includeQueries() throws Exception {
+    Thread t = new Thread(() -> {
+      try {
+        while (!testState.stopRunning.get()) {
+          int numQueryThreads = random().nextInt(MAX_NUM_OF_CONCURRENT_QUERY_REQUESTS_PER_ITERATION) + 1;
+          Thread[] indexingThreads = createQueryThreads(numQueryThreads);
+          for (int j = 0; j < numQueryThreads; j++) {
+            indexingThreads[j].start();
+          }
+          for (int j = 0; j < numQueryThreads; j++) {
+            indexingThreads[j].join();
+          }
+          Thread.sleep(DELAY_MS_BETWEEN_EACH_QUERY_ITERATION);
+        }
+      } catch (Exception ex) {
+        testState.queryErrors.add(ex.getMessage());
+      }
+    });
+    testState.includeThread(t);
+  }
 
-    if(interleaver != null) {
-      assertNull(interleaver.error, interleaver.error);
+  /**
+   * Creates {@code numQueryThreads} threads with each querying all docs "*:*"
+   */
+  private Thread[] createQueryThreads(int numQueryThreads) throws Exception {
+    log.info("numQueryThreads=" + numQueryThreads);
+    Thread[] queryThreads = new Thread[numQueryThreads];
+    for (int i = 0; i < numQueryThreads && !testState.stopRunning.get(); i++) {
+      queryThreads[i] = new Thread(() -> {
+        try {
+          /** 
+           * Don't have a way to ensure freshness of results yet. When we add something for query freshness later 
+           * we may use that here.
+           *
+           * {@link SolrProcessTracker#corePullTracker} cannot help in concurrent query scenarios since there
+           * is no one-to-one guarantee between query and an async pull.
+           */
+          cluster.getSolrClient().query(COLLECTION_NAME, new ModifiableSolrParams().set("q", "*:*"));
+        } catch (Exception ex) {
+          testState.queryErrors.add(Throwables.getStackTraceAsString(ex));
+        }
+      });
     }
+    return queryThreads;
+  }
 
-    assertFalse("no progress recorded", progress.isEmpty());
-
-    log.info(progress.toString());
-
-    assertCriticalSections(progress);
+  /**
+   * Adds a thread to test, that goes over iterations until the test is stopped {@link TestState#stopRunning}.
+   * In each iteration it failovers to new leader by calling {@link #failOver()}. It waits
+   * for {@link #DELAY_MS_BETWEEN_EACH_FAILOVER_ITERATION} between each iteration.
+   */
+  private void includeFailovers() throws Exception {
+    Thread t = new Thread(() -> {
+      try {
+        while (!testState.stopRunning.get()) {
+          failOver();
+          Thread.sleep(DELAY_MS_BETWEEN_EACH_FAILOVER_ITERATION);
+        }
+      } catch (Exception ex) {
+        testState.failoverError = Throwables.getStackTraceAsString(ex);
+      }
+    });
+    testState.includeThread(t);
+  }
 
-    // verify the update wasn't forwarded to the follower and it didn't commit by checking the core
-    // this gives us confidence that the subsequent query we do triggers the pull
-    CoreContainer replicaCC = getCoreContainer(followerReplica.getNodeName());
-    SolrCore core = null;
-    SolrClient followerDirectClient = null;
-    SolrClient leaderDirectClient = null;
-    try {
-      core = replicaCC.getCore(followerReplica.getCoreName());
-      // the follower should only have the default segments file
-      assertEquals(1, core.getDeletionPolicy().getLatestCommit().getFileNames().size());
+  /**
+   * Kills the current leader and waits for the new leader to be selected and then brings back up the killed leader
+   * as a follower replica. Before bringing back up the replica it randomly decides to delete its core directory.
+   */
+  private void failOver() throws Exception {
+    DocCollection collection = getCollection();
+    Replica leaderReplicaBeforeSwitch = collection.getLeader(SHARD_NAME);
+    final String leaderReplicaNameBeforeSwitch = leaderReplicaBeforeSwitch.getName();
+    JettySolrRunner shardLeaderSolrRunnerBeforeSwitch = cluster.getReplicaJetty(leaderReplicaBeforeSwitch);
+    File leaderIndexDirBeforeSwitch = new File(shardLeaderSolrRunnerBeforeSwitch.getCoreContainer().getCoreRootDirectory()
+        + "/" + leaderReplicaBeforeSwitch.getCoreName());
+
+    shardLeaderSolrRunnerBeforeSwitch.stop();
+    cluster.waitForJettyToStop(shardLeaderSolrRunnerBeforeSwitch);
+    waitForState("Timed out waiting for new replica to become leader", COLLECTION_NAME, (liveNodes, collectionState) -> {
+      Slice slice = collectionState.getSlice(SHARD_NAME);
+      if (slice.getLeader() == null) {
+        return false;
+      }
+      if (slice.getLeader().getName().equals(leaderReplicaNameBeforeSwitch)) {
+        return false;
+      }
 
-      // query the leader directly to verify it should have the document
-      leaderDirectClient = getHttpSolrClient(shardLeaderReplica.getBaseUrl() + "/" + shardLeaderReplica.getCoreName());
-      ModifiableSolrParams params = new ModifiableSolrParams();
-      params
-          .set("q", "*:*")
-          .set("distrib", "false");
-      QueryResponse resp = leaderDirectClient.query(params);
-      assertEquals(totalDocs.intValue(), resp.getResults().getNumFound());
-
-      // we want to wait until the pull completes so set up a count down latch for the follower's
-      // core that we'll wait until pull finishes for
-      CountDownLatch latch = new CountDownLatch(1);
-      Map<String, CountDownLatch> asyncPullTasks = solrProcessesTaskTracker.get(followerReplica.getNodeName());
-      asyncPullTasks.put(followerReplica.getCoreName(), latch);
+      return true;
+    });
 
-      // query the follower directly to trigger the pull, this query should yield no results
-      // as it returns immediately 
-      followerDirectClient = getHttpSolrClient(followerReplica.getBaseUrl() + "/" + followerReplica.getCoreName());
-      resp = followerDirectClient.query(params);
-      assertEquals(0, resp.getResults().getNumFound());
+    if (random().nextBoolean()) {
+      FileUtils.deleteDirectory(leaderIndexDirBeforeSwitch);
+    }
 
-      // wait until pull is finished
-      assertTrue(latch.await(120, TimeUnit.SECONDS));
+    shardLeaderSolrRunnerBeforeSwitch.start();
+    cluster.waitForNode(shardLeaderSolrRunnerBeforeSwitch, -1);
 
-      // do another query to verify we've pulled everything
-      resp = followerDirectClient.query(params);
+    waitForState("Timed out waiting for restarted replica to become active", COLLECTION_NAME, (liveNodes, collectionState) -> {
+      Slice slice = collectionState.getSlice(SHARD_NAME);
+      if (slice.getReplica(leaderReplicaNameBeforeSwitch).getState() != Replica.State.ACTIVE) {
+        return false;
+      }
+      return true;
+    });
 
-      // verify we pulled
-      assertTrue(core.getDeletionPolicy().getLatestCommit().getFileNames().size() > 1);
+    setupSolrProcess(shardLeaderSolrRunnerBeforeSwitch);
+  }
 
-      // verify the document is present
-      assertEquals(totalDocs.intValue(), resp.getResults().getNumFound());
-    } finally {
-      if (leaderDirectClient != null) {
-        leaderDirectClient.close();
-      }
-      if (followerDirectClient != null) {
-        followerDirectClient.close();
-      }
-      if (core != null) {
-        core.close();
-      }
-      if (cluster != null) {
-        cluster.shutdown();
+  /**
+   * Goes over all the lives of a node(node gets a new life on restart) and then goes over each core's concurrency stages
+   * in each life. Logs the concurrency stages in the order they occurred and then analyze those stages to make sure no
+   * critical section was breached.
+   */
+  private void analyzeCoreConcurrencyStagesForBreaches() {
+    // Goes over each node
+    for (Map.Entry<String, List<SolrProcessTracker>> nodeTracker :
+        testState.solrNodesTracker.entrySet()) {
+      String nodeName = nodeTracker.getKey();
+      int lifeCountForNode = nodeTracker.getValue().size();
+      // Goes over each life of a node
+      for (int i = 0; i < lifeCountForNode; i++) {
+        ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStageTracker = nodeTracker.getValue().get(i).coreConcurrencyStageTracker;
+        if (coreConcurrencyStageTracker.isEmpty()) {
+          log.info("life " + (i + 1) + "/" + lifeCountForNode + " of node " + nodeName + " is empty");
+        } else {
+          // Goes over each core
+          for (Map.Entry<String, ConcurrentLinkedQueue<String>> coreConcurrencyStagesEntry : coreConcurrencyStageTracker.entrySet()) {
+            String coreName = coreConcurrencyStagesEntry.getKey();
+            List<String> coreConcurrencyStages = new ArrayList<>(coreConcurrencyStagesEntry.getValue());
+            // Log line is truncated beyond certain length, therefore, printing them in the batches of 200
+            List<List<String>> batches = Lists.partition(coreConcurrencyStages, 200);
+            if (batches.isEmpty()) {
+              batches = new ArrayList<>(1);
+              batches.add(new ArrayList<>(0));
+            }
+            for (int j = 0; j < batches.size(); j++) {
+              log.info("batch " + (j + 1) + "/" + batches.size()
+                  + " of core " + coreName
+                  + " of life " + (i + 1) + "/" + lifeCountForNode
+                  + " of node " + nodeName
+                  + "\n" + batches.get(j).toString());
+            }
+            analyzeCoreConcurrencyStagesForBreaches(coreName, coreConcurrencyStages);
+          }
+        }
       }
-      // clean up the shared store. The temp dir should clean up itself after the test class finishes
-      FileUtils.cleanDirectory(sharedStoreRootPath.toFile());
     }
   }
 
-  private void assertCriticalSections(List<String> progress) {
-    String currentThreadId = null;
+  /**
+   * Analyze core's concurrency stages to make sure no critical section was breached. Detail of those critical sections
+   * can be found in {@link SharedCoreConcurrencyController}.
+   */
+  private void analyzeCoreConcurrencyStagesForBreaches(String coreName, List<String> coreConcurrencyStages) {
     SharedCoreStage currentStage = null;
-    String prevThreadId = null;
-    SharedCoreStage prevStage = null;
+    int activePullers = 0; // number of threads that have started pulling and not finished
     int activeIndexers = 0; // number of threads that have started indexing and not finished 
-    int activeBlobPushers = 0; // number of threads that are actively pushing at any given time
-    for (String p : progress) {
-      String[] parts = p.split("\\.");
-      currentThreadId = parts[0];
+    int activePushers = 0; // number of threads that are actively pushing at any given time
+    for (String s : coreConcurrencyStages) {
+      String[] parts = s.split("\\.");
       currentStage = SharedCoreStage.valueOf(parts[1]);
-      if (currentStage == SharedCoreStage.LocalIndexingStarted) {
+
+      if (currentStage == SharedCoreStage.BLOB_PULL_STARTED) {
+        activePullers++;
+      } else if (currentStage == SharedCoreStage.BLOB_PULL_FINISHED) {
+        activePullers--;
+      } else if (currentStage == SharedCoreStage.LOCAL_INDEXING_STARTED) {
         activeIndexers++;
-      } else if (currentStage == SharedCoreStage.BlobPushStarted) {
-        activeBlobPushers++;
-      } else if (currentStage == SharedCoreStage.BlobPushFinished) {
-        // both blob pushing and indexing finish at this stage 
-        activeBlobPushers--;
+      } else if (currentStage == SharedCoreStage.BLOB_PUSH_STARTED) {
+        activePushers++;
+      } else if (currentStage == SharedCoreStage.BLOB_PUSH_FINISHED) {
+        activePushers--;
+      } else if (currentStage == SharedCoreStage.INDEXING_BATCH_FINISHED) {
         activeIndexers--;
       }
 
-      // making sure no other activity takes place during pull
-      if (prevStage == SharedCoreStage.BlobPullStarted) {
-        assertTrue("Pull critical section breached, currentStage=" + p, prevThreadId==currentThreadId && currentStage == SharedCoreStage.BlobPullFinished);
-      }
-
-      // making sure indexing is not disrupted by a pull from blob
-      assertFalse("Indexing breached by a pull, currentStage=" + p,
-          activeIndexers > 0 &&
-              (currentStage == SharedCoreStage.BlobPullStarted || currentStage == SharedCoreStage.BlobPullFinished));
+      // making sure no other activity (including another pull) takes place during pull
+      assertFalse("Pull and indexing are interleaved, coreName=" + coreName + " currentStage=" + s, activePullers > 1 || (activePullers > 0 && (activeIndexers > 0 || activePushers > 0)));
 
       // making sure push to blob are not disrupted by another push to blob
-      assertFalse("Blob push breached by another blob push,  currentStage=" + p, activeBlobPushers > 1);
+      assertFalse("Blob push breached by another blob push, coreName=" + coreName + " currentStage=" + s, activePushers > 1);
+    }
+  }
 
-      prevThreadId = currentThreadId;
-      prevStage = currentStage;
+  /**
+   * Adds a new replica.
+   */
+  private Replica addReplica() throws Exception {
+    List<String> existingReplicas = getCollection().getSlice(SHARD_NAME).getReplicas().stream().map(r -> r.getName()).collect(Collectors.toList());
+    // add another replica
+    assertTrue(CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, SHARD_NAME, Replica.Type.SHARED)
+        .process(cluster.getSolrClient()).isSuccess());
+    // Verify that new replica is created
+    waitForState("Timed-out waiting for new replica to be created", COLLECTION_NAME, clusterShape(1, existingReplicas.size() + 1));
+
+    Replica newReplica = null;
+
+    for (Replica r : getCollection().getSlice(SHARD_NAME).getReplicas()) {
+      if (!existingReplicas.contains(r.getName())) {
+        newReplica = r;
+        break;
+      }
     }
+
+    assertNotNull("Could not find new replica", newReplica);
+
+    return newReplica;
   }
 
-  private void configureTestSharedConcurrencyControllerForNode(JettySolrRunner runner, List<String> progress) {
-    SharedCoreConcurrencyController concurrencyController = new SharedCoreConcurrencyController(runner.getCoreContainer()) {
-      Object recorderLock = new Object();
-      @Override
-      public void recordState(String collectionName, String shardName, String coreName, SharedCoreStage stage) {
-        super.recordState(collectionName, shardName, coreName, stage);
+  /**
+   * Directly query a new {@code replica} and verifies that the empty replica is correctly hydrated from the shared store
+   * with all the indexed docs (after accounting for deletions).
+   */
+  private void queryNewReplicaAndVerifyAllDocsFound(Replica replica) throws Exception {
+    try (SolrClient replicaDirectClient = getHttpSolrClient(replica.getBaseUrl() + "/" + replica.getCoreName())) {
+      ModifiableSolrParams params = new ModifiableSolrParams();
+      params.set("q", "*:*").set("distrib", "false").set("rows", testState.numDocsIndexed.get());
+      CountDownLatch latch = new CountDownLatch(1);
+      Map<String, CountDownLatch> corePullTracker = testState.getCorePullTracker(replica.getNodeName());
+      corePullTracker.put(replica.getCoreName(), latch);
+      QueryResponse resp = replicaDirectClient.query(params);
+      assertEquals("new replica did not return empty results", 0, resp.getResults().getNumFound());
 
-        synchronized (recorderLock) {
-          progress.add(Thread.currentThread().getId() + "." + stage.name());
-        }
+      assertTrue(latch.await(120, TimeUnit.SECONDS));
 
+      resp = replicaDirectClient.query(params);
+      List<String> docs = resp.getResults().stream().map(r -> (String) r.getFieldValue("id")).collect(Collectors.toList());
+      assertEquals("we did not ask for all the docs found", resp.getResults().getNumFound(), docs.size());
+      Collections.sort(docs, new Comparator<String>() {
+        public int compare(String id1, String id2) {
+          return Integer.parseInt(id1) - Integer.parseInt(id2);
+        }
+      });
+      List<String> docsExpected = new ArrayList<>();
+      for (int i = 1; i <= testState.numDocsIndexed.get(); i++) {
+        String docId = Integer.toString(i);
+        if (!testState.idsDeleted.contains(docId)) {
+          docsExpected.add(docId);
+        }
       }
-    };
-    setupTestSharedConcurrencyControllerForNode(concurrencyController, runner);
+      log.info("numDocsFound=" + docs.size() + " docsFound= " + docs.toString());
+      assertEquals("wrong docs", docsExpected.size() + docsExpected.toString(), docs.size() + docs.toString());
+    }
+  }
+
+  /**
+   * Setup all the nodes for test.
+   */
+  private void setupSolrNodesForTest() throws Exception {
+    for (JettySolrRunner solrProcess : cluster.getJettySolrRunners()) {
+      setupSolrProcess(solrProcess);
+    }
+  }
+
+  /**
+   * Setup solr process for test(process is one life of a node, restarts starts a new life).
+   */
+  private void setupSolrProcess(JettySolrRunner solrProcess) throws Exception {
+    CoreStorageClient storageClient = setupLocalBlobStoreClient(sharedStoreRootPath, DEFAULT_BLOB_DIR_NAME);
+    setupTestSharedClientForNode(getBlobStorageProviderTestInstance(storageClient), solrProcess);
+    Map<String, CountDownLatch> corePullTracker = configureTestBlobProcessForNode(solrProcess);
+    ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStagesTracker = new ConcurrentHashMap<>();
+    configureTestSharedConcurrencyControllerForProcess(solrProcess, coreConcurrencyStagesTracker);
+
+    SolrProcessTracker processTracker = new SolrProcessTracker(corePullTracker, coreConcurrencyStagesTracker);
+    List<SolrProcessTracker> nodeTracker = testState.solrNodesTracker.computeIfAbsent(solrProcess.getNodeName(), k -> new ArrayList<>());
+    nodeTracker.add(processTracker);
   }
 
-  private void configureTestSharedConcurrencyControllerForNode(JettySolrRunner runner, List<String> progress, IndexingThreadInterleaver interleaver) {
-    SharedCoreConcurrencyController concurrencyController = new SharedCoreConcurrencyController(runner.getCoreContainer()) {
-      Object recorderLock = new Object();
-      long firstThreadId = -1;
-      long secondThreadId = -1;
-      boolean firstAchieved = false;
-      boolean secondAchieved = false;
-      boolean secondAlreadyWaitingToStart = false;
-      CountDownLatch secondWaitingToStart = new CountDownLatch(1);
-      CountDownLatch firstWaitingForSecondToCompleteItsDesiredState = new CountDownLatch(1);
-      int timeoutSeconds = 10;
+  private DocCollection getCollection() {
+    return cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
+  }
+
+  /**
+   * Setup {@link SharedCoreConcurrencyController} for the solr process so we can accumulate concurrency stages a core
+   * goes through during test.
+   */
+  private void configureTestSharedConcurrencyControllerForProcess(
+      JettySolrRunner solrProcess, ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStagesMap) {
+    SharedCoreConcurrencyController concurrencyController = new SharedCoreConcurrencyController(solrProcess.getCoreContainer()) {
       @Override
       public void recordState(String collectionName, String shardName, String coreName, SharedCoreStage stage) {
         super.recordState(collectionName, shardName, coreName, stage);
+        ConcurrentLinkedQueue<String> coreConcurrencyStages = coreConcurrencyStagesMap.computeIfAbsent(coreName, k -> new ConcurrentLinkedQueue<>());
+        coreConcurrencyStages.add(Thread.currentThread().getId() + "." + stage.name());
+      }
+    };
+    setupTestSharedConcurrencyControllerForNode(concurrencyController, solrProcess);
+  }
 
-        CountDownLatch latchToWait = null;
-        boolean latchShouldSucceed = true;
-        String succeedError = null;
-        String failureError = null;
-        String interruptionError = null;
-        // compute stalling decision under synchronized block and then stall(if needed) outside of synchronized block
-        synchronized (recorderLock) {
-          progress.add(Thread.currentThread().getId() + "." + stage.name());
-
-          if (interleaver.error != null) {
-            // already errored out
-            return;
-          }
+  /**
+   * Manages state for each test from start to end.
+   */
+  private static class TestState {
+    /**
+     * Threads included in the test (indexing, queries and failovers).
+     */
+    private final List<Thread> includedThreads = new ArrayList<>();
+    /**
+     * Indicator when to stop. It is set to true when either indexing is done or interrupted.
+     */
+    private final AtomicBoolean stopRunning = new AtomicBoolean(false);
+    /**
+     * Used to provide unique id to each indexing doc.
+     */
+    private final AtomicInteger docIdGenerator = new AtomicInteger(0);
+    /**
+     * At any given moment how many minimum number of docs that have been indexed (it does not account for deletion)
+     */
+    private final AtomicInteger numDocsIndexed = new AtomicInteger(0);
+    /**
+     * Set of ids from indexed docs that can be deleted.
+     */
+    private final ConcurrentLinkedQueue<List<String>> idBatchesToDelete = new ConcurrentLinkedQueue<>();
+    /**
+     * Ids that have been deleted.
+     */
+    private final ConcurrentLinkedQueue<String> idsDeleted = new ConcurrentLinkedQueue<>();
+    /**
+     * Error setting up indexing or those encountered by indexing on the
+     * last attempt {@link #MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST} of each batch.
+     */
+    private final ConcurrentLinkedQueue<String> indexingErrors = new ConcurrentLinkedQueue<>();
+    /**
+     * Error setting up queries or those encountered by queries.
+     */
+    private final ConcurrentLinkedQueue<String> queryErrors = new ConcurrentLinkedQueue<>();
+    /**
+     * Error encountered when failing over to a new leader.
+     */
+    private String failoverError = null;
+    /**
+     * Tracks the cores' pull and concurrency stage information for each life of a node (node gets a new life on restart).
+     * Key is the node name.
+     */
+    private final Map<String, List<SolrProcessTracker>> solrNodesTracker = new HashMap<>();
 
-          long currentThreadId = Thread.currentThread().getId();
-          if (stage == SharedCoreStage.IndexingBatchReceived) {
-            // identify first and second thread and initialize their thread ids
-            // first is whichever comes first
-            if (firstThreadId == -1) {
-              firstThreadId = currentThreadId;
-            } else {
-              if(secondThreadId != -1) {
-                interleaver.error = "why there is a third indexing thread?";
-                return;
-              }
-              secondThreadId = currentThreadId;
-            }
-          }
+    /**
+     * Gets the core pull tracker for current life of the node.
+     */
+    private Map<String, CountDownLatch> getCorePullTracker(String nodeName) {
+      List<SolrProcessTracker> allLives = solrNodesTracker.get(nodeName);
+      return allLives.get(allLives.size() - 1).corePullTracker;
+    }
 
-          if (firstAchieved && secondAchieved) {
-            // nothing left
-            return;
-          } else if (!secondAlreadyWaitingToStart && currentThreadId == secondThreadId) {
-            secondAlreadyWaitingToStart = true;
-            latchToWait = secondWaitingToStart;
-            latchShouldSucceed = true;
-            failureError = "second failed to wait for first to reach at desired state";
-            interruptionError = "second was interrupted while waiting for first to reach at desired state";
-          } else if (currentThreadId == firstThreadId && stage == interleaver.firstStage) {
-            // first reached to desired stage, now release second
-            secondWaitingToStart.countDown();
-            latchToWait = firstWaitingForSecondToCompleteItsDesiredState;
-            latchShouldSucceed = interleaver.isPossible;
-            succeedError = "first was able to wait for second to reach at desired state, even when it was not possible";
-            failureError = "first failed to wait for second to reach at desired state";
-            interruptionError = "first was interrupted while waiting for second to reach at desired state";
-            firstAchieved = true;
-          } else if (currentThreadId == secondThreadId && stage == interleaver.secondStage) {
-            // second also reached desired state, we are done
-            if (!firstAchieved) {
-              interleaver.error = "how come second reached its desired without first reaching its";
-            }
-            firstWaitingForSecondToCompleteItsDesiredState.countDown();
-            secondAchieved = true;
-          }
-        }
+    /**
+     * Includes a thread into test.
+     */
+    private void includeThread(Thread t) {
+      includedThreads.add(t);
+    }
 
-        if (latchToWait != null) {
-          // need to stall this stage on the provided latch
-          try {
-            boolean successfulWait = latchToWait.await(timeoutSeconds, TimeUnit.SECONDS);
-            if (successfulWait && !latchShouldSucceed) {
-              interleaver.error = succeedError;
-            } else if (!successfulWait && latchShouldSucceed) {
-              interleaver.error = failureError;
-            }
-          } catch (InterruptedException iex) {
-            interleaver.error = interruptionError;
-          }
-        }
+    /**
+     * Starts all the included threads.
+     */
+    private void startIncludedThreads() throws Exception {
+      for (Thread t : includedThreads) {
+        t.start();
       }
-    };
-    setupTestSharedConcurrencyControllerForNode(concurrencyController, runner);
+    }
+
+    /**
+     * Wait for all the included threads to stop.
+     */
+    private void waitForThreadsToStop() throws Exception {
+      for (Thread t : includedThreads) {
+        t.join();
+      }
+      log.info("docIdGenerator=" + docIdGenerator.get() + " numDocsIndexed=" + numDocsIndexed.get() + " numDocsDeleted=" + idsDeleted.size());
+    }
+
+    /**
+     * Check if any error was encountered during the test.
+     */
+    private void checkErrors() {
+      assertTrue("indexingErrors=\n" + indexingErrors.toString() + "\n"
+              + "queryErrors=\n" + queryErrors.toString() + "\n"
+              + "failoverError=\n" + failoverError + "\n",
+          indexingErrors.isEmpty() && queryErrors.isEmpty() && failoverError == null);
+    }
   }
 
   /**
-   * 1. Second thread is stalled after {@link SharedCoreStage#IndexingBatchReceived} 
-   * 2. First thread is run up to firstStage then stalled
-   * 3. Second thread is resumed and run up to secondStage
-   * 4. First thread is resumed
+   * Track cores' pull and concurrency stage information for one life of a node
    */
-  private static class IndexingThreadInterleaver {
-    private final SharedCoreStage firstStage;
+  private static class SolrProcessTracker {
     /**
-     * Let second indexing thread run up to this stage before resuming stalled first thread
+     * Per core pull tracker.
+     * Key is the core name.
      */
-    private final SharedCoreStage secondStage;
-    private final boolean isPossible;
-    private String error = null;
-
-    private IndexingThreadInterleaver(SharedCoreStage firstStage,
-                                      SharedCoreStage secondStage,
-                                      boolean isPossible) {
-      this.firstStage = firstStage;
-      this.secondStage = secondStage;
-      this.isPossible = isPossible;
+    private final Map<String, CountDownLatch> corePullTracker;
+    /**
+     * Per core concurrency stage tracker.
+     * Key is the core name.
+     * 
+     * For now we are only using single replica per node therefore it will only be single core
+     * but it should be able to handle multiple replicas per node, if test chooses to setup that way.
+     */
+    private final ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStageTracker;
+
+    private SolrProcessTracker(Map<String, CountDownLatch> corePullTracker,
+                               ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStageTracker) {
+      this.corePullTracker = corePullTracker;
+      this.coreConcurrencyStageTracker = coreConcurrencyStageTracker;
     }
   }
 }