You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2019/12/21 14:38:39 UTC
[lucene-solr] branch jira/SOLR-13101 updated: SOLR-13101:
Concurrency tests for SHARED collection. (#1081)
This is an automated email from the ASF dual-hosted git repository.
yonik pushed a commit to branch jira/SOLR-13101
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-13101 by this push:
new 036ce53 SOLR-13101: Concurrency tests for SHARED collection. (#1081)
036ce53 is described below
commit 036ce539525aa9c0eb0270d94e4105cd1a0313e9
Author: Bilal Waheed <23...@users.noreply.github.com>
AuthorDate: Sat Dec 21 06:38:26 2019 -0800
SOLR-13101: Concurrency tests for SHARED collection. (#1081)
* SOLR-13101: Concurrency tests for SHARED collection.
-Concurrency tests for SHARED collection.
-On core container shutdown, shutdown the background blob delete manager too.
-Use correct casing for SharedCoreStage enum.
-Added INDEXING_BATCH_FINISHED to indicate the end of a batch (BLOB_PUSH_FINISHED does not help if push itself runs into error)
* -CR feedback.
---
.../java/org/apache/solr/core/CoreContainer.java | 1 +
.../solr/store/blob/process/CorePullTask.java | 4 +-
.../apache/solr/store/blob/process/CorePusher.java | 10 +-
.../solr/store/blob/util/BlobStoreUtils.java | 4 +-
.../shared/SharedCoreConcurrencyController.java | 24 +-
.../processor/DistributedZkUpdateProcessor.java | 15 +-
.../store/shared/SharedCoreConcurrencyTest.java | 913 ++++++++++++++-------
7 files changed, 630 insertions(+), 341 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 56c8354..db05a55 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1071,6 +1071,7 @@ public class CoreContainer {
if (sharedStoreManager != null) {
sharedStoreManager.getBlobProcessManager().shutdown();
+ sharedStoreManager.getBlobDeleteManager().shutdown();
}
}
diff --git a/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java b/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java
index f100a4d..44eccca 100644
--- a/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java
+++ b/solr/core/src/java/org/apache/solr/store/blob/process/CorePullTask.java
@@ -243,7 +243,7 @@ public class CorePullTask implements DeduplicatingList.Deduplicatable<String> {
return;
}
- concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BlobPullStarted);
+ concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BLOB_PULL_STARTED);
// Get blob metadata
String blobCoreMetadataName = BlobStoreUtils.buildBlobStoreMetadataName(shardVersionMetadata.getMetadataSuffix());
blobMetadata = blobClient.pullCoreMetadata(pullCoreInfo.getSharedStoreName(), blobCoreMetadataName);
@@ -351,7 +351,7 @@ public class CorePullTask implements DeduplicatingList.Deduplicatable<String> {
}
}
this.callback.finishedPull(this, blobMetadata, syncStatus, message);
- concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BlobPullFinished);
+ concurrencyController.recordState(pullCoreInfo.getCollectionName(), pullCoreInfo.getShardName(), pullCoreInfo.getCoreName(), SharedCoreStage.BLOB_PULL_FINISHED);
}
void finishedPull(BlobCoreMetadata blobCoreMetadata, CoreSyncStatus syncStatus, String message) throws InterruptedException {
diff --git a/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java b/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java
index 2712b8d..2f6e215 100644
--- a/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java
+++ b/solr/core/src/java/org/apache/solr/store/blob/process/CorePusher.java
@@ -85,7 +85,7 @@ public class CorePusher {
throw new SolrException(ErrorCode.SERVER_ERROR, "Can't find core " + coreName);
}
try {
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPushStarted);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PUSH_STARTED);
IndexCommit latestCommit = core.getDeletionPolicy().getLatestCommit();
if (latestCommit == null) {
@@ -130,7 +130,7 @@ public class CorePusher {
// begin the push process
CorePushPull pushPull = new CorePushPull(blobClient, deleteManager, pushPullData, resolutionResult, localCoreMetadata, coreVersionMetadata.getBlobCoreMetadata());
BlobCoreMetadata blobCoreMetadata = pushPull.pushToBlobStore(coreVersionMetadata.getMetadataSuffix(), newMetadataSuffix);
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPushed);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PUSHED);
// at this point we've pushed the new metadata file with the newMetadataSuffix and now need to write to zookeeper
SharedShardMetadataController shardSharedMetadataController = coreContainer.getSharedStoreManager().getSharedShardMetadataController();
SharedShardVersionMetadata newShardVersionMetadata = null;
@@ -154,17 +154,17 @@ public class CorePusher {
}
throw ex;
}
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.ZkUpdateFinished);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.ZK_UPDATE_FINISHED);
assert newMetadataSuffix.equals(newShardVersionMetadata.getMetadataSuffix());
// after successful update to zookeeper, update core version metadata with new version info
// and we can also give soft guarantee that core is up to date w.r.to shared store, until unless failures happen and leadership changes
concurrencyController.updateCoreVersionMetadata(collectionName, shardName, coreName, newShardVersionMetadata, blobCoreMetadata, /* softGuaranteeOfEquality */ true);
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LocalCacheUpdateFinished);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LOCAL_CACHE_UPDATE_FINISHED);
log.info(String.format("Successfully pushed to shared store, pushLockTime=%s pushPullData=%s", lockAcquisitionTime, pushPullData.toString()));
} finally {
try {
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPushFinished);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PUSH_FINISHED);
if (snapshotDirPath != null) {
// we are done with push we can now remove the snapshot directory
removeSnapshotDirectory(core, snapshotDirPath);
diff --git a/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java b/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java
index a9e26e5..c31ea79 100644
--- a/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java
+++ b/solr/core/src/java/org/apache/solr/store/blob/util/BlobStoreUtils.java
@@ -83,7 +83,7 @@ public class BlobStoreUtils {
log.info("sync successful, nothing to pull, collection=" + collectionName + " shard=" + shardName + " coreName=" + coreName);
return;
}
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPullStarted);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PULL_STARTED);
try {
// Get blob metadata
String blobCoreMetadataName = BlobStoreUtils.buildBlobStoreMetadataName(shardVersionMetadata.getMetadataSuffix());
@@ -128,7 +128,7 @@ public class BlobStoreUtils {
collectionName, shardName, coreName));
}
} finally {
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BlobPullFinished);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.BLOB_PULL_FINISHED);
}
} catch (Exception ex) {
// wrap every thrown exception in a solr exception
diff --git a/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java b/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java
index 2adde86..06c8714 100644
--- a/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java
+++ b/solr/core/src/java/org/apache/solr/store/shared/SharedCoreConcurrencyController.java
@@ -390,43 +390,47 @@ public class SharedCoreConcurrencyController {
/**
* Necessary locks have been acquired and we have started to pull from the shared store.
*/
- BlobPullStarted,
+ BLOB_PULL_STARTED,
/**
* Pull(either successful or failed) has ended and we are about to release the necessary locks.
*/
- BlobPullFinished,
+ BLOB_PULL_FINISHED,
/**
* We have received an indexing batch but necessary locks have not been acquired yet.
*/
- IndexingBatchReceived,
+ INDEXING_BATCH_RECEIVED,
/**
* We are passed the shared store pull stage(if applicable) and are in sync with shared store.
* Now we will proceed with local indexing.
*/
- LocalIndexingStarted,
+ LOCAL_INDEXING_STARTED,
/**
* Local indexing finished but not have been pushed to shared store.
*/
- LocalIndexingFinished,
+ LOCAL_INDEXING_FINISHED,
/**
* Necessary locks have been acquired and push to shared store has started.
*/
- BlobPushStarted,
+ BLOB_PUSH_STARTED,
/**
* Files have been pushed to blob.
*/
- BlobPushed,
+ BLOB_PUSHED,
/**
* Zookeeper has been successfully updated with new metadata.
*/
- ZkUpdateFinished,
+ ZK_UPDATE_FINISHED,
/**
* Local cache {@link #coresVersionMetadata} has been successfully updated with new metadata.
*/
- LocalCacheUpdateFinished,
+ LOCAL_CACHE_UPDATE_FINISHED,
/**
* Push(either successful or failed) has ended and we are about to release the necessary locks.
*/
- BlobPushFinished
+ BLOB_PUSH_FINISHED,
+ /**
+ * Indexing batch(either successful or failed) has ended and we are about to release the necessary locks.
+ */
+ INDEXING_BATCH_FINISHED
}
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index 5a8abdb..f06845b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -1067,6 +1067,14 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
@Override
protected void doClose() {
if (corePullLock != null) {
+ try {
+ SharedCoreConcurrencyController concurrencyController =
+ req.getCore().getCoreContainer().getSharedStoreManager().getSharedCoreConcurrencyController();
+ concurrencyController.recordState(cloudDesc.getCollectionName(), cloudDesc.getShardId(), req.getCore().getName(),
+ SharedCoreStage.INDEXING_BATCH_FINISHED);
+ } catch (Exception ex) {
+ SolrException.log(log, "Error recording the finish of SHARED core indexing batch", ex);
+ }
// release read lock
corePullLock.readLock().unlock();
}
@@ -1097,12 +1105,13 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
}
+ // TODO: Refactor most of shared replica logic out into a separate class.
private void writeToSharedStore() {
String collectionName = cloudDesc.getCollectionName();
String shardName = cloudDesc.getShardId();
String coreName = req.getCore().getName();
SharedCoreConcurrencyController concurrencyController = req.getCore().getCoreContainer().getSharedStoreManager().getSharedCoreConcurrencyController();
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LocalIndexingFinished);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LOCAL_INDEXING_FINISHED);
log.info("Attempting to initiate index update write to shared store for collection=" + collectionName +
" and shard=" + shardName + " using core=" + coreName);
@@ -1134,7 +1143,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
CoreContainer coreContainer = req.getCore().getCoreContainer();
SharedCoreConcurrencyController concurrencyController = coreContainer.getSharedStoreManager().getSharedCoreConcurrencyController();
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.IndexingBatchReceived);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.INDEXING_BATCH_RECEIVED);
corePullLock = concurrencyController.getCorePullLock(collectionName, shardName, coreName);
// from this point on wards we should always exit this method with read lock (no matter failure or what)
try {
@@ -1155,7 +1164,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// acquire lock for the whole duration of update
// we should always leave with read lock acquired(failure or success), since it is the job of close method to release it
corePullLock.readLock().lock();
- concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LocalIndexingStarted);
+ concurrencyController.recordState(collectionName, shardName, coreName, SharedCoreStage.LOCAL_INDEXING_STARTED);
}
}
diff --git a/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java b/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java
index 8ec1fad..7753d9f 100644
--- a/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java
+++ b/solr/core/src/test/org/apache/solr/store/shared/SharedCoreConcurrencyTest.java
@@ -16,31 +16,40 @@
*/
package org.apache.solr.store.shared;
+import java.io.File;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrCore;
import org.apache.solr.store.blob.client.CoreStorageClient;
import org.apache.solr.store.shared.SharedCoreConcurrencyController.SharedCoreStage;
+import org.junit.After;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -49,411 +58,677 @@ import org.slf4j.LoggerFactory;
/**
* Tests around synchronization of concurrent indexing, pushes and pulls
* happening on a core of a shared collection {@link DocCollection#getSharedIndex()}
- * todo: add tests for failover scenarios and involve query pulls
*/
public class SharedCoreConcurrencyTest extends SolrCloudSharedStoreTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+ private static final String COLLECTION_NAME = "sharedCollection";
+ private static final String SHARD_NAME = "shard1";
+ /**
+ * Number of serial indexing iterations for each test. This is the main setting, queries and failover iterations
+ * stop after indexing ends. Higher the value, longer the tests would run.
+ */
+ private static int INDEXING_ITERATIONS = TEST_NIGHTLY ? 100 : 20;
+ /**
+ * Maximum number of concurrent indexing requests per indexing iteration.
+ */
+ private static int MAX_NUM_OF_CONCURRENT_INDEXING_REQUESTS_PER_ITERATION = 10;
+ /**
+ * Maximum number of docs per indexing request.
+ */
+ private static int MAX_NUM_OF_DOCS_PER_INDEXING_REQUEST = 100;
+ /**
+ * Indexing can fail because of leader failures (especially when test {@link #includeFailovers()}).
+ * The test will re-attempt up till this number of times before bailing out. For test to succeed,
+ * indexing request have to succeed in these many attempts.
+ */
+ private static int MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST = 10;
+ /**
+ * Maximum number of concurrent query requests per query iteration.
+ */
+ private static int MAX_NUM_OF_CONCURRENT_QUERY_REQUESTS_PER_ITERATION = 10;
+ /**
+ * Querying is faster than indexing, to pace it better with indexing, a delay is added between each query iteration.
+ */
+ private static int DELAY_MS_BETWEEN_EACH_QUERY_ITERATION = 50;
+ /**
+ * Minimum time between each failover.
+ */
+ private static int DELAY_MS_BETWEEN_EACH_FAILOVER_ITERATION = 500;
+ /**
+ * Path for local shared store
+ */
private static Path sharedStoreRootPath;
+ /**
+ * Manages test state from start to end.
+ */
+ private TestState testState;
+
@BeforeClass
public static void setupClass() throws Exception {
sharedStoreRootPath = createTempDir("tempDir");
}
+ @Before
+ public void setupTest() throws Exception {
+ int numNodes = 4;
+ setupCluster(numNodes);
+ testState = new TestState();
+ setupSolrNodesForTest();
+
+ int maxShardsPerNode = 1;
+ // One less than number of nodes.
+ // The extra node will be used at the end of test to verify
+ // the contents of shared store by querying for all docs on a new replica.
+ int numReplicas = numNodes - 1;
+ // Later on we can consider choosing random number of shards and replicas.
+ // To handle multiple shards, we need to update code where SHARD_NAME is used.
+ setupSharedCollectionWithShardNames(COLLECTION_NAME, maxShardsPerNode, numReplicas, SHARD_NAME);
+ }
+
+ @After
+ public void teardownTest() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ FileUtils.cleanDirectory(sharedStoreRootPath.toFile());
+ }
+
/**
- * Tests issuing random number of concurrent indexing requests in a given range for a shared core and making sure they all succeed.
+ * Tests that concurrent indexing succeed.
*/
@Test
- public void testHighConcurrentIndexing() throws Exception {
- int maxIndexingThreads = 100;
- int maxDocsPerThread = 100;
- testConcurrentIndexing(maxIndexingThreads, maxDocsPerThread);
+ public void testIndexing() throws Exception {
+ final boolean includeDeletes = false;
+ includeIndexing(includeDeletes);
+ run();
}
/**
- * Test ensuring two indexing requests interleave in desired ways and succeed.
+ * Tests that concurrent indexing with concurrent queries succeed.
*/
- // @Test todo: leaking threads, test only issue
- public void TODOtestPossibleInterleaving() throws Exception {
-
- // completely serialized
- testConcurrentIndexing(
- new IndexingThreadInterleaver(
- SharedCoreStage.BlobPushFinished,
- SharedCoreStage.LocalIndexingStarted,
- true));
-
- // second pushes for first one too
- testConcurrentIndexing(
- new IndexingThreadInterleaver(
- SharedCoreStage.LocalIndexingFinished,
- SharedCoreStage.BlobPushFinished,
- true));
+ @Test
+ public void testIndexingQueries() throws Exception {
+ final boolean includeDeletes = false;
+ includeIndexing(includeDeletes);
+ includeQueries();
+ run();
}
/**
- * Test ensuring two indexing requests do not interleave in impossible ways even when forced and still succeed.
+ * Tests that concurrent indexing with deletes and concurrent queries succeed.
*/
- // @Test todo: leaking threads, test only issue
- public void TODOtestImpossibleInterleaving() throws Exception {
- // push critical section
- testConcurrentIndexing(
- new IndexingThreadInterleaver(
- SharedCoreStage.ZkUpdateFinished,
- SharedCoreStage.ZkUpdateFinished,
- false));
-
- // another push critical section
- testConcurrentIndexing(
- new IndexingThreadInterleaver(
- SharedCoreStage.ZkUpdateFinished,
- SharedCoreStage.LocalCacheUpdateFinished,
- false));
+ @Test
+ public void testIndexingQueriesDeletes() throws Exception {
+ final boolean includeDeletes = true;
+ includeIndexing(includeDeletes);
+ includeQueries();
+ run();
}
- private void testConcurrentIndexing(int maxIndexingThreads, int maxDocsPerThread) throws Exception {
- int numIndexingThreads = new Random().nextInt(maxIndexingThreads) + 1;;
- testConcurrentIndexing(numIndexingThreads, maxDocsPerThread, null);
+ /**
+ * Tests that concurrent indexing with deletes, concurrent queries and explicit failovers succeed.
+ */
+ // @Test
+ // TODO: This test flaps time to time. The symptom of the failure is missing docs i.e. indexing is declared successful
+ // but query could not reproduce all of the docs. I was able to repro this with NRT collection on vanilla 8.3 too.
+ // I have not root caused it yet. Keeping this test disabled, until the problem is root caused and fixed.
+ public void todoTestIndexingQueriesDeletesFailovers() throws Exception {
+ final boolean includeDeletes = true;
+ includeIndexing(includeDeletes);
+ includeQueries();
+ includeFailovers();
+ run();
}
- private void testConcurrentIndexing(IndexingThreadInterleaver interleaver) throws Exception {
- testConcurrentIndexing(2, 10, interleaver);
+ /**
+ * It starts all the threads that are included in the test (indexing, queries and failovers) in parallel.
+ * Then wait for them to finish (run length depends on {@link #INDEXING_ITERATIONS}).
+ * At the end it makes sures that no critical section was breached and no unexpected error occurred.
+ * Then verify the contents of shared store by querying for all docs on a new replica.
+ */
+ private void run() throws Exception {
+ testState.startIncludedThreads();
+ testState.waitForThreadsToStop();
+ analyzeCoreConcurrencyStagesForBreaches();
+ testState.checkErrors();
+ Replica newReplica = addReplica();
+ queryNewReplicaAndVerifyAllDocsFound(newReplica);
}
/**
- * Start desired number of concurrent indexing threads with each indexing random number of docs between 1 and maxDocsPerThread.
+ * Adds a thread to test, that goes over {@link #INDEXING_ITERATIONS} or until it is interrupted.
+ * In each iteration it creates between 1 and {@link #MAX_NUM_OF_CONCURRENT_INDEXING_REQUESTS_PER_ITERATION} threads
+ * by calling {@link #createIndexingThreads(int, int, boolean)}, starts them concurrently and wait for them to finish
+ * before going to next iteration. Each indexing thread adds between 1 and {@link #MAX_NUM_OF_DOCS_PER_INDEXING_REQUEST}
+ * docs.
*
- * At the end it verifies everything got indexed successfully on leader. No critical section got breached.
- * Also verify the integrity of shared store contents by pulling them on a follower replica.
+ * @param includeDeletes whether to randomly mark some docs for deletion and delete them in subsequent indexing requests
+ * or not
*/
- private void testConcurrentIndexing(int numIndexingThreads, int maxDocsPerThread, IndexingThreadInterleaver interleaver) throws Exception {
- setupCluster(2);
- CloudSolrClient cloudClient = cluster.getSolrClient();
-
- // this map tracks the async pull queues per solr process
- Map<String, Map<String, CountDownLatch>> solrProcessesTaskTracker = new HashMap<>();
-
- JettySolrRunner solrProcess1 = cluster.getJettySolrRunner(0);
- CoreStorageClient storageClient1 = setupLocalBlobStoreClient(sharedStoreRootPath, DEFAULT_BLOB_DIR_NAME);
- setupTestSharedClientForNode(getBlobStorageProviderTestInstance(storageClient1), solrProcess1);
- Map<String, CountDownLatch> asyncPullLatches1 = configureTestBlobProcessForNode(solrProcess1);
-
- JettySolrRunner solrProcess2 = cluster.getJettySolrRunner(1);
- CoreStorageClient storageClient2 = setupLocalBlobStoreClient(sharedStoreRootPath, DEFAULT_BLOB_DIR_NAME);
- setupTestSharedClientForNode(getBlobStorageProviderTestInstance(storageClient2), solrProcess2);
- Map<String, CountDownLatch> asyncPullLatches2 = configureTestBlobProcessForNode(solrProcess2);
-
- solrProcessesTaskTracker.put(solrProcess1.getNodeName(), asyncPullLatches1);
- solrProcessesTaskTracker.put(solrProcess2.getNodeName(), asyncPullLatches2);
-
- String collectionName = "sharedCollection";
- int maxShardsPerNode = 1;
- int numReplicas = 2;
- // specify a comma-delimited string of shard names for multiple shards when using
- // an implicit router
- String shardNames = "shard1";
- setupSharedCollectionWithShardNames(collectionName, maxShardsPerNode, numReplicas, shardNames);
-
- // get the leader replica and follower replicas
- DocCollection collection = cloudClient.getZkStateReader().getClusterState().getCollection(collectionName);
- Replica shardLeaderReplica = collection.getLeader("shard1");
- Replica followerReplica = null;
- for (Replica repl : collection.getSlice("shard1").getReplicas()) {
- if (repl.getName() != shardLeaderReplica.getName()) {
- followerReplica = repl;
- break;
+ private void includeIndexing(boolean includeDeletes) {
+ Thread t = new Thread(() -> {
+ try {
+ for (int i = 0; i < INDEXING_ITERATIONS && !testState.stopRunning.get(); i++) {
+ int numIndexingThreads = random().nextInt(MAX_NUM_OF_CONCURRENT_INDEXING_REQUESTS_PER_ITERATION) + 1;
+ int numDocsToAddPerThread = random().nextInt(MAX_NUM_OF_DOCS_PER_INDEXING_REQUEST) + 1;
+ Thread[] indexingThreads = createIndexingThreads(numIndexingThreads, numDocsToAddPerThread, includeDeletes);
+ for (int j = 0; j < numIndexingThreads; j++) {
+ indexingThreads[j].start();
+ }
+ for (int j = 0; j < numIndexingThreads; j++) {
+ indexingThreads[j].join();
+ }
+ if (Thread.interrupted()) {
+ // we have been interrupted so we will stop running
+ testState.stopRunning.set(true);
+ }
+ }
+ } catch (Exception ex) {
+ testState.indexingErrors.add(ex.getMessage());
}
- }
+ // everything else stops running when indexing finishes
+ testState.stopRunning.set(true);
+ });
+ testState.includeThread(t);
+ }
-
- JettySolrRunner shardLeaderSolrRunner = null;
- for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
- if(solrRunner.getNodeName().equals(shardLeaderReplica.getNodeName())){
- shardLeaderSolrRunner = solrRunner;
- break;
- }
- }
- List<String> progress = new ArrayList<>();
-
- if(interleaver != null) {
- configureTestSharedConcurrencyControllerForNode(shardLeaderSolrRunner, progress, interleaver);
- } else {
- configureTestSharedConcurrencyControllerForNode(shardLeaderSolrRunner, progress);
- }
-
- AtomicInteger totalDocs= new AtomicInteger(0);
+ /**
+ * Creates {@code numIndexingThreads} threads with each adding {@code numDocsToAddPerThread}.
+ *
+ * @param includeDeletes whether to randomly mark some docs for deletion and delete them in subsequent indexing requests
+ * or not
+ */
+ private Thread[] createIndexingThreads(int numIndexingThreads, int numDocsToAddPerThread, boolean includeDeletes) throws Exception {
log.info("numIndexingThreads=" + numIndexingThreads);
Thread[] indexingThreads = new Thread[numIndexingThreads];
- ConcurrentLinkedQueue<String> indexingErrors = new ConcurrentLinkedQueue<>();
- for (int i = 0; i < numIndexingThreads; i++) {
+ for (int i = 0; i < numIndexingThreads && !testState.stopRunning.get(); i++) {
indexingThreads[i] = new Thread(() -> {
- try {
- // index between 1 to maxDocsPerThread docs
- int numDocs = new Random().nextInt(maxDocsPerThread) + 1;
- log.info("numDocs=" + numDocs);
- UpdateRequest updateReq = new UpdateRequest();
- for (int j = 0; j < numDocs; j++) {
- int docId = totalDocs.incrementAndGet();
- updateReq.add("id", Integer.toString(docId));
+ List<String> idsToAdd = new ArrayList<>();
+ // prepare the list of docs to add and delete outside the reattempt loop
+ for (int j = 0; j < numDocsToAddPerThread; j++) {
+ String docId = Integer.toString(testState.docIdGenerator.incrementAndGet());
+ idsToAdd.add(docId);
+ }
+ List<String> idsToDelete = testState.idBatchesToDelete.poll();
+
+ // attempt until succeeded or max attempts
+ for (int j = 0; j < MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST; j++) {
+ try {
+ String message = "attempt=" + (j + 1) + " numDocsToAdd=" + numDocsToAddPerThread + " docsToAdd=" + idsToAdd.toString();
+ if (idsToDelete != null) {
+ message += " numDocsToDelete=" + idsToDelete.size() + " docsToDelete=" + idsToDelete.toString();
+ }
+ log.info(message);
+
+ UpdateRequest updateReq = new UpdateRequest();
+ for (int k = 0; k < idsToAdd.size(); k++) {
+ updateReq.add("id", idsToAdd.get(k));
+ }
+ if (includeDeletes && idsToDelete != null) {
+ updateReq.deleteById(idsToDelete);
+ }
+ processUpdateRequest(updateReq);
+
+ testState.numDocsIndexed.addAndGet(numDocsToAddPerThread);
+ if (idsToDelete != null) {
+ testState.idsDeleted.addAll(idsToDelete);
+ }
+
+ // randomly select some docs that can be deleted
+ if (includeDeletes) {
+ List<String> idsThatCanBeDeleted = new ArrayList<>();
+ for (String indexedId : idsToAdd) {
+ if (random().nextBoolean()) {
+ idsThatCanBeDeleted.add(indexedId);
+ }
+ }
+ if (!idsThatCanBeDeleted.isEmpty()) {
+ testState.idBatchesToDelete.offer(idsThatCanBeDeleted);
+ }
+ }
+ // indexing was successful, stop attempting
+ break;
+ } catch (Exception ex) {
+ // last attempt also failed, record the error
+ if (j == MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST - 1) {
+ testState.indexingErrors.add(Throwables.getStackTraceAsString(ex));
+ }
}
- updateReq.commit(cloudClient, collectionName);
- } catch (Exception ex) {
- indexingErrors.add(ex.getMessage());
}
});
}
+ return indexingThreads;
+ }
- for (int i = 0; i < numIndexingThreads; i++) {
- indexingThreads[i].start();
- }
+ /**
+ * Sends update request to the server, randomly choosing whether to send it with commit=true or not
+ * Shared replica does not need an explicit commit since it always does an implicit hard commit but
+ * still it is valid to send an update with or without a commit, therefore, testing both.
+ */
+ private void processUpdateRequest(UpdateRequest request) throws Exception {
+ UpdateResponse response = random().nextBoolean()
+ ? request.process(cluster.getSolrClient(), COLLECTION_NAME)
+ : request.commit(cluster.getSolrClient(), COLLECTION_NAME);
- for (int i = 0; i < numIndexingThreads; i++) {
- indexingThreads[i].join();
+ if (response.getStatus() != 0) {
+ throw new RuntimeException("Update request failed with status=" + response.getStatus());
}
+ }
- log.info("totalDocs=" + totalDocs.intValue());
-
- assertTrue(indexingErrors.toString(), indexingErrors.isEmpty());
+ /**
+ * Adds a thread to test, that goes over iterations until the test is stopped {@link TestState#stopRunning}.
+ * In each iteration it creates between 1 and {@link #MAX_NUM_OF_CONCURRENT_QUERY_REQUESTS_PER_ITERATION} threads
+ * by calling {@link #createQueryThreads(int)}, starts them concurrently and wait for them to finish
+ * before going to next iteration. To pace it better with indexing, {@link #DELAY_MS_BETWEEN_EACH_QUERY_ITERATION}
+ * delay is added between each query iteration.
+ */
+ private void includeQueries() throws Exception {
+ Thread t = new Thread(() -> {
+ try {
+ while (!testState.stopRunning.get()) {
+ int numQueryThreads = random().nextInt(MAX_NUM_OF_CONCURRENT_QUERY_REQUESTS_PER_ITERATION) + 1;
+ Thread[] indexingThreads = createQueryThreads(numQueryThreads);
+ for (int j = 0; j < numQueryThreads; j++) {
+ indexingThreads[j].start();
+ }
+ for (int j = 0; j < numQueryThreads; j++) {
+ indexingThreads[j].join();
+ }
+ Thread.sleep(DELAY_MS_BETWEEN_EACH_QUERY_ITERATION);
+ }
+ } catch (Exception ex) {
+ testState.queryErrors.add(ex.getMessage());
+ }
+ });
+ testState.includeThread(t);
+ }
- if(interleaver != null) {
- assertNull(interleaver.error, interleaver.error);
+ /**
+ * Creates {@code numQueryThreads} threads with each querying all docs "*:*"
+ */
+ private Thread[] createQueryThreads(int numQueryThreads) throws Exception {
+ log.info("numQueryThreads=" + numQueryThreads);
+ Thread[] queryThreads = new Thread[numQueryThreads];
+ for (int i = 0; i < numQueryThreads && !testState.stopRunning.get(); i++) {
+ queryThreads[i] = new Thread(() -> {
+ try {
+ /**
+ * Don't have a way to ensure freshness of results yet. When we add something for query freshness later
+ * we may use that here.
+ *
+ * {@link SolrProcessTracker#corePullTracker} cannot help in concurrent query scenarios since there
+ * is no one-to-one guarantee between query and an async pull.
+ */
+ cluster.getSolrClient().query(COLLECTION_NAME, new ModifiableSolrParams().set("q", "*:*"));
+ } catch (Exception ex) {
+ testState.queryErrors.add(Throwables.getStackTraceAsString(ex));
+ }
+ });
}
+ return queryThreads;
+ }
- assertFalse("no progress recorded", progress.isEmpty());
-
- log.info(progress.toString());
-
- assertCriticalSections(progress);
+ /**
+ * Adds a thread to test, that goes over iterations until the test is stopped {@link TestState#stopRunning}.
+ * In each iteration it failovers to new leader by calling {@link #failOver()}. It waits
+ * for {@link #DELAY_MS_BETWEEN_EACH_FAILOVER_ITERATION} between each iteration.
+ */
+ private void includeFailovers() throws Exception {
+ Thread t = new Thread(() -> {
+ try {
+ while (!testState.stopRunning.get()) {
+ failOver();
+ Thread.sleep(DELAY_MS_BETWEEN_EACH_FAILOVER_ITERATION);
+ }
+ } catch (Exception ex) {
+ testState.failoverError = Throwables.getStackTraceAsString(ex);
+ }
+ });
+ testState.includeThread(t);
+ }
- // verify the update wasn't forwarded to the follower and it didn't commit by checking the core
- // this gives us confidence that the subsequent query we do triggers the pull
- CoreContainer replicaCC = getCoreContainer(followerReplica.getNodeName());
- SolrCore core = null;
- SolrClient followerDirectClient = null;
- SolrClient leaderDirectClient = null;
- try {
- core = replicaCC.getCore(followerReplica.getCoreName());
- // the follower should only have the default segments file
- assertEquals(1, core.getDeletionPolicy().getLatestCommit().getFileNames().size());
+ /**
+ * Kills the current leader and waits for the new leader to be selected and then brings back up the killed leader
+ * as a follower replica. Before bringing back up the replica it randomly decides to delete its core directory.
+ */
+ private void failOver() throws Exception {
+ DocCollection collection = getCollection();
+ Replica leaderReplicaBeforeSwitch = collection.getLeader(SHARD_NAME);
+ final String leaderReplicaNameBeforeSwitch = leaderReplicaBeforeSwitch.getName();
+ JettySolrRunner shardLeaderSolrRunnerBeforeSwitch = cluster.getReplicaJetty(leaderReplicaBeforeSwitch);
+ File leaderIndexDirBeforeSwitch = new File(shardLeaderSolrRunnerBeforeSwitch.getCoreContainer().getCoreRootDirectory()
+ + "/" + leaderReplicaBeforeSwitch.getCoreName());
+
+ shardLeaderSolrRunnerBeforeSwitch.stop();
+ cluster.waitForJettyToStop(shardLeaderSolrRunnerBeforeSwitch);
+ waitForState("Timed out waiting for new replica to become leader", COLLECTION_NAME, (liveNodes, collectionState) -> {
+ Slice slice = collectionState.getSlice(SHARD_NAME);
+ if (slice.getLeader() == null) {
+ return false;
+ }
+ if (slice.getLeader().getName().equals(leaderReplicaNameBeforeSwitch)) {
+ return false;
+ }
- // query the leader directly to verify it should have the document
- leaderDirectClient = getHttpSolrClient(shardLeaderReplica.getBaseUrl() + "/" + shardLeaderReplica.getCoreName());
- ModifiableSolrParams params = new ModifiableSolrParams();
- params
- .set("q", "*:*")
- .set("distrib", "false");
- QueryResponse resp = leaderDirectClient.query(params);
- assertEquals(totalDocs.intValue(), resp.getResults().getNumFound());
-
- // we want to wait until the pull completes so set up a count down latch for the follower's
- // core that we'll wait until pull finishes for
- CountDownLatch latch = new CountDownLatch(1);
- Map<String, CountDownLatch> asyncPullTasks = solrProcessesTaskTracker.get(followerReplica.getNodeName());
- asyncPullTasks.put(followerReplica.getCoreName(), latch);
+ return true;
+ });
- // query the follower directly to trigger the pull, this query should yield no results
- // as it returns immediately
- followerDirectClient = getHttpSolrClient(followerReplica.getBaseUrl() + "/" + followerReplica.getCoreName());
- resp = followerDirectClient.query(params);
- assertEquals(0, resp.getResults().getNumFound());
+ if (random().nextBoolean()) {
+ FileUtils.deleteDirectory(leaderIndexDirBeforeSwitch);
+ }
- // wait until pull is finished
- assertTrue(latch.await(120, TimeUnit.SECONDS));
+ shardLeaderSolrRunnerBeforeSwitch.start();
+ cluster.waitForNode(shardLeaderSolrRunnerBeforeSwitch, -1);
- // do another query to verify we've pulled everything
- resp = followerDirectClient.query(params);
+ waitForState("Timed out waiting for restarted replica to become active", COLLECTION_NAME, (liveNodes, collectionState) -> {
+ Slice slice = collectionState.getSlice(SHARD_NAME);
+ if (slice.getReplica(leaderReplicaNameBeforeSwitch).getState() != Replica.State.ACTIVE) {
+ return false;
+ }
+ return true;
+ });
- // verify we pulled
- assertTrue(core.getDeletionPolicy().getLatestCommit().getFileNames().size() > 1);
+ setupSolrProcess(shardLeaderSolrRunnerBeforeSwitch);
+ }
- // verify the document is present
- assertEquals(totalDocs.intValue(), resp.getResults().getNumFound());
- } finally {
- if (leaderDirectClient != null) {
- leaderDirectClient.close();
- }
- if (followerDirectClient != null) {
- followerDirectClient.close();
- }
- if (core != null) {
- core.close();
- }
- if (cluster != null) {
- cluster.shutdown();
+ /**
+ * Goes over all the lives of a node(node gets a new life on restart) and then goes over each core's concurrency stages
+ * in each life. Logs the concurrency stages in the order they occurred and then analyze those stages to make sure no
+ * critical section was breached.
+ */
+ private void analyzeCoreConcurrencyStagesForBreaches() {
+ // Goes over each node
+ for (Map.Entry<String, List<SolrProcessTracker>> nodeTracker :
+ testState.solrNodesTracker.entrySet()) {
+ String nodeName = nodeTracker.getKey();
+ int lifeCountForNode = nodeTracker.getValue().size();
+ // Goes over each life of a node
+ for (int i = 0; i < lifeCountForNode; i++) {
+ ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStageTracker = nodeTracker.getValue().get(i).coreConcurrencyStageTracker;
+ if (coreConcurrencyStageTracker.isEmpty()) {
+ log.info("life " + (i + 1) + "/" + lifeCountForNode + " of node " + nodeName + " is empty");
+ } else {
+ // Goes over each core
+ for (Map.Entry<String, ConcurrentLinkedQueue<String>> coreConcurrencyStagesEntry : coreConcurrencyStageTracker.entrySet()) {
+ String coreName = coreConcurrencyStagesEntry.getKey();
+ List<String> coreConcurrencyStages = new ArrayList<>(coreConcurrencyStagesEntry.getValue());
+ // Log line is truncated beyond certain length, therefore, printing them in the batches of 200
+ List<List<String>> batches = Lists.partition(coreConcurrencyStages, 200);
+ if (batches.isEmpty()) {
+ batches = new ArrayList<>(1);
+ batches.add(new ArrayList<>(0));
+ }
+ for (int j = 0; j < batches.size(); j++) {
+ log.info("batch " + (j + 1) + "/" + batches.size()
+ + " of core " + coreName
+ + " of life " + (i + 1) + "/" + lifeCountForNode
+ + " of node " + nodeName
+ + "\n" + batches.get(j).toString());
+ }
+ analyzeCoreConcurrencyStagesForBreaches(coreName, coreConcurrencyStages);
+ }
+ }
}
- // clean up the shared store. The temp dir should clean up itself after the test class finishes
- FileUtils.cleanDirectory(sharedStoreRootPath.toFile());
}
}
- private void assertCriticalSections(List<String> progress) {
- String currentThreadId = null;
+ /**
+ * Analyze core's concurrency stages to make sure no critical section was breached. Detail of those critical sections
+ * can be found in {@link SharedCoreConcurrencyController}.
+ */
+ private void analyzeCoreConcurrencyStagesForBreaches(String coreName, List<String> coreConcurrencyStages) {
SharedCoreStage currentStage = null;
- String prevThreadId = null;
- SharedCoreStage prevStage = null;
+ int activePullers = 0; // number of threads that have started pulling and not finished
int activeIndexers = 0; // number of threads that have started indexing and not finished
- int activeBlobPushers = 0; // number of threads that are actively pushing at any given time
- for (String p : progress) {
- String[] parts = p.split("\\.");
- currentThreadId = parts[0];
+ int activePushers = 0; // number of threads that are actively pushing at any given time
+ for (String s : coreConcurrencyStages) {
+ String[] parts = s.split("\\.");
currentStage = SharedCoreStage.valueOf(parts[1]);
- if (currentStage == SharedCoreStage.LocalIndexingStarted) {
+
+ if (currentStage == SharedCoreStage.BLOB_PULL_STARTED) {
+ activePullers++;
+ } else if (currentStage == SharedCoreStage.BLOB_PULL_FINISHED) {
+ activePullers--;
+ } else if (currentStage == SharedCoreStage.LOCAL_INDEXING_STARTED) {
activeIndexers++;
- } else if (currentStage == SharedCoreStage.BlobPushStarted) {
- activeBlobPushers++;
- } else if (currentStage == SharedCoreStage.BlobPushFinished) {
- // both blob pushing and indexing finish at this stage
- activeBlobPushers--;
+ } else if (currentStage == SharedCoreStage.BLOB_PUSH_STARTED) {
+ activePushers++;
+ } else if (currentStage == SharedCoreStage.BLOB_PUSH_FINISHED) {
+ activePushers--;
+ } else if (currentStage == SharedCoreStage.INDEXING_BATCH_FINISHED) {
activeIndexers--;
}
- // making sure no other activity takes place during pull
- if (prevStage == SharedCoreStage.BlobPullStarted) {
- assertTrue("Pull critical section breached, currentStage=" + p, prevThreadId==currentThreadId && currentStage == SharedCoreStage.BlobPullFinished);
- }
-
- // making sure indexing is not disrupted by a pull from blob
- assertFalse("Indexing breached by a pull, currentStage=" + p,
- activeIndexers > 0 &&
- (currentStage == SharedCoreStage.BlobPullStarted || currentStage == SharedCoreStage.BlobPullFinished));
+ // making sure no other activity (including another pull) takes place during pull
+ assertFalse("Pull and indexing are interleaved, coreName=" + coreName + " currentStage=" + s, activePullers > 1 || (activePullers > 0 && (activeIndexers > 0 || activePushers > 0)));
// making sure push to blob are not disrupted by another push to blob
- assertFalse("Blob push breached by another blob push, currentStage=" + p, activeBlobPushers > 1);
+ assertFalse("Blob push breached by another blob push, coreName=" + coreName + " currentStage=" + s, activePushers > 1);
+ }
+ }
- prevThreadId = currentThreadId;
- prevStage = currentStage;
+ /**
+ * Adds a new replica.
+ */
+ private Replica addReplica() throws Exception {
+ List<String> existingReplicas = getCollection().getSlice(SHARD_NAME).getReplicas().stream().map(r -> r.getName()).collect(Collectors.toList());
+ // add another replica
+ assertTrue(CollectionAdminRequest.addReplicaToShard(COLLECTION_NAME, SHARD_NAME, Replica.Type.SHARED)
+ .process(cluster.getSolrClient()).isSuccess());
+ // Verify that new replica is created
+ waitForState("Timed-out waiting for new replica to be created", COLLECTION_NAME, clusterShape(1, existingReplicas.size() + 1));
+
+ Replica newReplica = null;
+
+ for (Replica r : getCollection().getSlice(SHARD_NAME).getReplicas()) {
+ if (!existingReplicas.contains(r.getName())) {
+ newReplica = r;
+ break;
+ }
}
+
+ assertNotNull("Could not find new replica", newReplica);
+
+ return newReplica;
}
- private void configureTestSharedConcurrencyControllerForNode(JettySolrRunner runner, List<String> progress) {
- SharedCoreConcurrencyController concurrencyController = new SharedCoreConcurrencyController(runner.getCoreContainer()) {
- Object recorderLock = new Object();
- @Override
- public void recordState(String collectionName, String shardName, String coreName, SharedCoreStage stage) {
- super.recordState(collectionName, shardName, coreName, stage);
+ /**
+ * Directly query a new {@code replica} and verifies that the empty replica is correctly hydrated from the shared store
+ * with all the indexed docs (after accounting for deletions).
+ */
+ private void queryNewReplicaAndVerifyAllDocsFound(Replica replica) throws Exception {
+ try (SolrClient replicaDirectClient = getHttpSolrClient(replica.getBaseUrl() + "/" + replica.getCoreName())) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("q", "*:*").set("distrib", "false").set("rows", testState.numDocsIndexed.get());
+ CountDownLatch latch = new CountDownLatch(1);
+ Map<String, CountDownLatch> corePullTracker = testState.getCorePullTracker(replica.getNodeName());
+ corePullTracker.put(replica.getCoreName(), latch);
+ QueryResponse resp = replicaDirectClient.query(params);
+ assertEquals("new replica did not return empty results", 0, resp.getResults().getNumFound());
- synchronized (recorderLock) {
- progress.add(Thread.currentThread().getId() + "." + stage.name());
- }
+ assertTrue(latch.await(120, TimeUnit.SECONDS));
+ resp = replicaDirectClient.query(params);
+ List<String> docs = resp.getResults().stream().map(r -> (String) r.getFieldValue("id")).collect(Collectors.toList());
+ assertEquals("we did not ask for all the docs found", resp.getResults().getNumFound(), docs.size());
+ Collections.sort(docs, new Comparator<String>() {
+ public int compare(String id1, String id2) {
+ return Integer.parseInt(id1) - Integer.parseInt(id2);
+ }
+ });
+ List<String> docsExpected = new ArrayList<>();
+ for (int i = 1; i <= testState.numDocsIndexed.get(); i++) {
+ String docId = Integer.toString(i);
+ if (!testState.idsDeleted.contains(docId)) {
+ docsExpected.add(docId);
+ }
}
- };
- setupTestSharedConcurrencyControllerForNode(concurrencyController, runner);
+ log.info("numDocsFound=" + docs.size() + " docsFound= " + docs.toString());
+ assertEquals("wrong docs", docsExpected.size() + docsExpected.toString(), docs.size() + docs.toString());
+ }
+ }
+
+ /**
+ * Setup all the nodes for test.
+ */
+ private void setupSolrNodesForTest() throws Exception {
+ for (JettySolrRunner solrProcess : cluster.getJettySolrRunners()) {
+ setupSolrProcess(solrProcess);
+ }
+ }
+
+ /**
+ * Setup solr process for test(process is one life of a node, restarts starts a new life).
+ */
+ private void setupSolrProcess(JettySolrRunner solrProcess) throws Exception {
+ CoreStorageClient storageClient = setupLocalBlobStoreClient(sharedStoreRootPath, DEFAULT_BLOB_DIR_NAME);
+ setupTestSharedClientForNode(getBlobStorageProviderTestInstance(storageClient), solrProcess);
+ Map<String, CountDownLatch> corePullTracker = configureTestBlobProcessForNode(solrProcess);
+ ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStagesTracker = new ConcurrentHashMap<>();
+ configureTestSharedConcurrencyControllerForProcess(solrProcess, coreConcurrencyStagesTracker);
+
+ SolrProcessTracker processTracker = new SolrProcessTracker(corePullTracker, coreConcurrencyStagesTracker);
+ List<SolrProcessTracker> nodeTracker = testState.solrNodesTracker.computeIfAbsent(solrProcess.getNodeName(), k -> new ArrayList<>());
+ nodeTracker.add(processTracker);
}
- private void configureTestSharedConcurrencyControllerForNode(JettySolrRunner runner, List<String> progress, IndexingThreadInterleaver interleaver) {
- SharedCoreConcurrencyController concurrencyController = new SharedCoreConcurrencyController(runner.getCoreContainer()) {
- Object recorderLock = new Object();
- long firstThreadId = -1;
- long secondThreadId = -1;
- boolean firstAchieved = false;
- boolean secondAchieved = false;
- boolean secondAlreadyWaitingToStart = false;
- CountDownLatch secondWaitingToStart = new CountDownLatch(1);
- CountDownLatch firstWaitingForSecondToCompleteItsDesiredState = new CountDownLatch(1);
- int timeoutSeconds = 10;
+ private DocCollection getCollection() {
+ return cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME);
+ }
+
+ /**
+ * Setup {@link SharedCoreConcurrencyController} for the solr process so we can accumulate concurrency stages a core
+ * goes through during test.
+ */
+ private void configureTestSharedConcurrencyControllerForProcess(
+ JettySolrRunner solrProcess, ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStagesMap) {
+ SharedCoreConcurrencyController concurrencyController = new SharedCoreConcurrencyController(solrProcess.getCoreContainer()) {
@Override
public void recordState(String collectionName, String shardName, String coreName, SharedCoreStage stage) {
super.recordState(collectionName, shardName, coreName, stage);
+ ConcurrentLinkedQueue<String> coreConcurrencyStages = coreConcurrencyStagesMap.computeIfAbsent(coreName, k -> new ConcurrentLinkedQueue<>());
+ coreConcurrencyStages.add(Thread.currentThread().getId() + "." + stage.name());
+ }
+ };
+ setupTestSharedConcurrencyControllerForNode(concurrencyController, solrProcess);
+ }
- CountDownLatch latchToWait = null;
- boolean latchShouldSucceed = true;
- String succeedError = null;
- String failureError = null;
- String interruptionError = null;
- // compute stalling decision under synchronized block and then stall(if needed) outside of synchronized block
- synchronized (recorderLock) {
- progress.add(Thread.currentThread().getId() + "." + stage.name());
-
- if (interleaver.error != null) {
- // already errored out
- return;
- }
+ /**
+ * Manages state for each test from start to end.
+ */
+ private static class TestState {
+ /**
+ * Threads included in the test (indexing, queries and failovers).
+ */
+ private final List<Thread> includedThreads = new ArrayList<>();
+ /**
+ * Indicator when to stop. It is set to true when either indexing is done or interrupted.
+ */
+ private final AtomicBoolean stopRunning = new AtomicBoolean(false);
+ /**
+ * Used to provide unique id to each indexing doc.
+ */
+ private final AtomicInteger docIdGenerator = new AtomicInteger(0);
+ /**
+ * At any given moment how many minimum number of docs that have been indexed (it does not account for deletion)
+ */
+ private final AtomicInteger numDocsIndexed = new AtomicInteger(0);
+ /**
+ * Set of ids from indexed docs that can be deleted.
+ */
+ private final ConcurrentLinkedQueue<List<String>> idBatchesToDelete = new ConcurrentLinkedQueue<>();
+ /**
+ * Ids that have been deleted.
+ */
+ private final ConcurrentLinkedQueue<String> idsDeleted = new ConcurrentLinkedQueue<>();
+ /**
+ * Error setting up indexing or those encountered by indexing on the
+ * last attempt {@link #MAX_NUM_OF_ATTEMPTS_PER_INDEXING_REQUEST} of each batch.
+ */
+ private final ConcurrentLinkedQueue<String> indexingErrors = new ConcurrentLinkedQueue<>();
+ /**
+ * Error setting up queries or those encountered by queries.
+ */
+ private final ConcurrentLinkedQueue<String> queryErrors = new ConcurrentLinkedQueue<>();
+ /**
+ * Error encountered when failing over to a new leader.
+ */
+ private String failoverError = null;
+ /**
+ * Tracks the cores' pull and concurrency stage information for each life of a node (node gets a new life on restart).
+ * Key is the node name.
+ */
+ private final Map<String, List<SolrProcessTracker>> solrNodesTracker = new HashMap<>();
- long currentThreadId = Thread.currentThread().getId();
- if (stage == SharedCoreStage.IndexingBatchReceived) {
- // identify first and second thread and initialize their thread ids
- // first is whichever comes first
- if (firstThreadId == -1) {
- firstThreadId = currentThreadId;
- } else {
- if(secondThreadId != -1) {
- interleaver.error = "why there is a third indexing thread?";
- return;
- }
- secondThreadId = currentThreadId;
- }
- }
+ /**
+ * Gets the core pull tracker for current life of the node.
+ */
+ private Map<String, CountDownLatch> getCorePullTracker(String nodeName) {
+ List<SolrProcessTracker> allLives = solrNodesTracker.get(nodeName);
+ return allLives.get(allLives.size() - 1).corePullTracker;
+ }
- if (firstAchieved && secondAchieved) {
- // nothing left
- return;
- } else if (!secondAlreadyWaitingToStart && currentThreadId == secondThreadId) {
- secondAlreadyWaitingToStart = true;
- latchToWait = secondWaitingToStart;
- latchShouldSucceed = true;
- failureError = "second failed to wait for first to reach at desired state";
- interruptionError = "second was interrupted while waiting for first to reach at desired state";
- } else if (currentThreadId == firstThreadId && stage == interleaver.firstStage) {
- // first reached to desired stage, now release second
- secondWaitingToStart.countDown();
- latchToWait = firstWaitingForSecondToCompleteItsDesiredState;
- latchShouldSucceed = interleaver.isPossible;
- succeedError = "first was able to wait for second to reach at desired state, even when it was not possible";
- failureError = "first failed to wait for second to reach at desired state";
- interruptionError = "first was interrupted while waiting for second to reach at desired state";
- firstAchieved = true;
- } else if (currentThreadId == secondThreadId && stage == interleaver.secondStage) {
- // second also reached desired state, we are done
- if (!firstAchieved) {
- interleaver.error = "how come second reached its desired without first reaching its";
- }
- firstWaitingForSecondToCompleteItsDesiredState.countDown();
- secondAchieved = true;
- }
- }
+ /**
+ * Includes a thread into test.
+ */
+ private void includeThread(Thread t) {
+ includedThreads.add(t);
+ }
- if (latchToWait != null) {
- // need to stall this stage on the provided latch
- try {
- boolean successfulWait = latchToWait.await(timeoutSeconds, TimeUnit.SECONDS);
- if (successfulWait && !latchShouldSucceed) {
- interleaver.error = succeedError;
- } else if (!successfulWait && latchShouldSucceed) {
- interleaver.error = failureError;
- }
- } catch (InterruptedException iex) {
- interleaver.error = interruptionError;
- }
- }
+ /**
+ * Starts all the included threads.
+ */
+ private void startIncludedThreads() throws Exception {
+ for (Thread t : includedThreads) {
+ t.start();
}
- };
- setupTestSharedConcurrencyControllerForNode(concurrencyController, runner);
+ }
+
+ /**
+ * Wait for all the included threads to stop.
+ */
+ private void waitForThreadsToStop() throws Exception {
+ for (Thread t : includedThreads) {
+ t.join();
+ }
+ log.info("docIdGenerator=" + docIdGenerator.get() + " numDocsIndexed=" + numDocsIndexed.get() + " numDocsDeleted=" + idsDeleted.size());
+ }
+
+ /**
+ * Check if any error was encountered during the test.
+ */
+ private void checkErrors() {
+ assertTrue("indexingErrors=\n" + indexingErrors.toString() + "\n"
+ + "queryErrors=\n" + queryErrors.toString() + "\n"
+ + "failoverError=\n" + failoverError + "\n",
+ indexingErrors.isEmpty() && queryErrors.isEmpty() && failoverError == null);
+ }
}
/**
- * 1. Second thread is stalled after {@link SharedCoreStage#IndexingBatchReceived}
- * 2. First thread is run up to firstStage then stalled
- * 3. Second thread is resumed and run up to secondStage
- * 4. First thread is resumed
+ * Track cores' pull and concurrency stage information for one life of a node
*/
- private static class IndexingThreadInterleaver {
- private final SharedCoreStage firstStage;
+ private static class SolrProcessTracker {
/**
- * Let second indexing thread run up to this stage before resuming stalled first thread
+ * Per core pull tracker.
+ * Key is the core name.
*/
- private final SharedCoreStage secondStage;
- private final boolean isPossible;
- private String error = null;
-
- private IndexingThreadInterleaver(SharedCoreStage firstStage,
- SharedCoreStage secondStage,
- boolean isPossible) {
- this.firstStage = firstStage;
- this.secondStage = secondStage;
- this.isPossible = isPossible;
+ private final Map<String, CountDownLatch> corePullTracker;
+ /**
+ * Per core concurrency stage tracker.
+ * Key is the core name.
+ *
+ * For now we are only using single replica per node therefore it will only be single core
+ * but it should be able to handle multiple replicas per node, if test chooses to setup that way.
+ */
+ private final ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStageTracker;
+
+ private SolrProcessTracker(Map<String, CountDownLatch> corePullTracker,
+ ConcurrentHashMap<String, ConcurrentLinkedQueue<String>> coreConcurrencyStageTracker) {
+ this.corePullTracker = corePullTracker;
+ this.coreConcurrencyStageTracker = coreConcurrencyStageTracker;
}
}
}