You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2016/10/04 12:49:34 UTC
[2/2] lucene-solr:branch_6x: SOLR-9563: Collection creation can fail
if a node doesn't update its state quickly enough
SOLR-9563: Collection creation can fail if a node doesn't update its state quickly enough
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1e378150
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1e378150
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1e378150
Branch: refs/heads/branch_6x
Commit: 1e3781500c238e0c15b588137205ba5f1c560f58
Parents: 23485c3
Author: Alan Woodward <ro...@apache.org>
Authored: Tue Oct 4 12:26:17 2016 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Tue Oct 4 13:48:55 2016 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 4 ++
.../org/apache/solr/cloud/ZkController.java | 52 ++++++++++++--------
.../solr/cloud/DeleteInactiveReplicaTest.java | 2 +-
3 files changed, 36 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1e378150/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4b13a5b..534d0fb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -121,6 +121,10 @@ Bug Fixes
* SOLR-9556: OverseerAutoFailoverReplicaThread was not exiting on interrupt
(Alan Woodward)
+* SOLR-9563: Collection creation could fail if an ADDREPLICA subrequest arrived
+ at a node before its local state had updated with the new collection data
+ (Alan Woodward)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1e378150/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index ea85a67..b56d1c9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -43,6 +43,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
@@ -1524,34 +1525,43 @@ public class ZkController {
if (!Overseer.isLegacy(zkStateReader)) {
CloudDescriptor cloudDesc = cd.getCloudDescriptor();
String coreNodeName = cloudDesc.getCoreNodeName();
- assert coreNodeName != null : "SolrCore: " + cd.getName() + " has no coreNodeName";
+ if (coreNodeName == null)
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No coreNodeName for " + cd);
if (cloudDesc.getShardId() == null) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "No shard id for :" + cd);
- }
- long endTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS);
- String errMessage = null;
- while (System.nanoTime() < endTime) {
- Slice slice = zkStateReader.getClusterState().getSlice(cd.getCollectionName(), cloudDesc.getShardId());
- if (slice == null) {
- errMessage = "Invalid slice : " + cloudDesc.getShardId();
- continue;
- }
- if (slice.getReplica(coreNodeName) != null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No shard id for " + cd);
+ }
+
+ AtomicReference<String> errorMessage = new AtomicReference<>();
+ AtomicReference<DocCollection> collectionState = new AtomicReference<>();
+ try {
+ zkStateReader.waitForState(cd.getCollectionName(), 3, TimeUnit.SECONDS, (n, c) -> {
+ collectionState.set(c);
+ if (c == null)
+ return false;
+ Slice slice = c.getSlice(cloudDesc.getShardId());
+ if (slice == null) {
+ errorMessage.set("Invalid shard: " + cloudDesc.getShardId());
+ return false;
+ }
Replica replica = slice.getReplica(coreNodeName);
+ if (replica == null) {
+ errorMessage.set("coreNodeName " + coreNodeName + " does not exist in shard " + cloudDesc.getShardId());
+ return false;
+ }
String baseUrl = replica.getStr(BASE_URL_PROP);
String coreName = replica.getStr(CORE_NAME_PROP);
if (baseUrl.equals(this.baseURL) && coreName.equals(cd.getName())) {
- return;
- } else {
- errMessage = "replica with coreNodeName " + coreNodeName + " exists but with a different name or base_url";
+ return true;
}
- }
- Thread.sleep(100);
- }
- if (errMessage == null) {
- errMessage = "replica " + coreNodeName + " is not present in cluster state";
+ errorMessage.set("coreNodeName " + coreNodeName + " exists, but does not match expected node or core name");
+ return false;
+ });
+ } catch (TimeoutException e) {
+ String error = errorMessage.get();
+ if (error == null)
+ error = "Replica " + coreNodeName + " is not present in cluster state";
+ throw new SolrException(ErrorCode.SERVER_ERROR, error + ": " + collectionState.get());
}
- throw new SolrException(ErrorCode.SERVER_ERROR, errMessage + ". state : " + zkStateReader.getClusterState().getCollection(cd.getCollectionName()));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1e378150/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
index 91608d5..8f8731b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteInactiveReplicaTest.java
@@ -85,7 +85,7 @@ public class DeleteInactiveReplicaTest extends SolrCloudTestCase {
CoreContainer.CoreLoadFailure loadFailure = cc.getCoreInitFailures().get(replica.getCoreName());
assertNotNull("Deleted core was still loaded!", loadFailure);
assertTrue("Unexpected load failure message: " + loadFailure.exception.getMessage(),
- loadFailure.exception.getMessage().contains("not present in cluster state"));
+ loadFailure.exception.getMessage().contains("does not exist in shard"));
// Check that we can't create a core with no coreNodeName
try (SolrClient queryClient = getHttpSolrClient(jetty.getBaseUrl().toString())) {