You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/10/13 16:50:55 UTC
[lucene-solr] branch reference_impl_dev updated: @985 Update how we
track collection zk versions in Overseer.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl_dev by this push:
new 1cdd19d @985 Update how we track collection zk versions in Overseer.
1cdd19d is described below
commit 1cdd19d0bfc181f74edcd90fbd63b8aeded7aa26
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Oct 13 11:49:03 2020 -0500
@985 Update how we track collection zk versions in Overseer.
---
.../apache/solr/cloud/overseer/ZkStateWriter.java | 70 ++++++++++------------
.../solr/cloud/MissingSegmentRecoveryTest.java | 2 -
.../apache/solr/common/cloud/ZkStateReader.java | 6 +-
.../solr/common/util/SolrQueuedThreadPool.java | 29 +++++----
4 files changed, 50 insertions(+), 57 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 32a1ead..4aecdc9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud.overseer;
import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -63,6 +64,7 @@ public class ZkStateWriter {
// / protected boolean isClusterStateModified = false;
protected long lastUpdatedTime = 0;
+ private Map<String,Integer> trackVersions = new HashMap<>();
private final ZkStateReader reader;
@@ -126,11 +128,11 @@ public class ZkStateWriter {
}
updatesToWrite.remove(name);
+ trackVersions.remove(name);
reader.getZkClient().clean(path);
TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- DocCollection coll;
timeout.waitFor("", () -> {
DocCollection rc = reader.getClusterState().getCollectionOrNull(name);
if (rc == null) return true;
@@ -140,17 +142,11 @@ public class ZkStateWriter {
LinkedHashMap<String,ClusterState.CollectionRef> collStates = new LinkedHashMap<>(prevState.getCollectionStates());
collStates.remove(name);
prevState = new ClusterState(prevState.getLiveNodes(), collStates, prevState.getZNodeVersion());
- } else if (updatesToWrite.get(name) != null || prevState.getCollectionOrNull(name) != null) {
+ } else if (updatesToWrite.get(name) != null) {
if (log.isDebugEnabled()) {
log.debug("enqueueUpdate() - going to update_collection {} version: {}", path, c.getZNodeVersion());
}
- // assert c.getStateFormat() > 1;
- // stat = reader.getZkClient().getCurator().checkExists().forPath(path);
- DocCollection coll = updatesToWrite.get(name);
- if (coll == null) {
- coll = prevState.getCollectionOrNull(name);
- }
if (log.isDebugEnabled()) {
log.debug("The new collection {}", c);
@@ -230,6 +226,7 @@ public class ZkStateWriter {
//
//
Map<String,DocCollection> failedUpdates = new LinkedHashMap<>();
+ Exception lastFailedException = null;
for (DocCollection c : updatesToWrite.values()) {
String name = c.getName();
String path = ZkStateReader.getCollectionPath(c.getName());
@@ -247,21 +244,24 @@ public class ZkStateWriter {
//}
// stat = reader.getZkClient().getCurator().setData().withVersion(prevVersion).forPath(path, data);
try {
- stat = reader.getZkClient().setData(path, data, c.getZNodeVersion(), false);
- } catch (KeeperException.BadVersionException bve) {
-
- if (c.getZNodeVersion() == 1 && stat.getVersion() == 0) {
- // need to figure out how this case happens
- stat = reader.getZkClient().setData(path, data, 0, false);
+ int version = c.getZNodeVersion();
+ Integer v = trackVersions.get(c.getName());
+ if (v != null && version == 0) {
+ version = v;
+ trackVersions.put(c.getName(), v + 1);
} else {
- // this is a tragic error, we must disallow usage of this instance
- log.warn(
- "Tried to update the cluster state using version={} but we where rejected, found {}",
- c.getZNodeVersion(), stat.getVersion(), bve);
- throw bve;
+ trackVersions.put(c.getName(), version + 1);
}
- }
+ reader.getZkClient().setData(path, data, version, true);
+ } catch (KeeperException.BadVersionException bve) {
+ lastFailedException = bve;
+ failedUpdates.put(c.getName(), c);
+ stat = reader.getZkClient().exists(path, null);
+ // this is a tragic error, we must disallow usage of this instance
+ log.warn("Tried to update the cluster state using version={} but we where rejected, found {}", c.getZNodeVersion(), stat.getVersion(), bve);
+ }
+ if (log.isDebugEnabled()) log.debug("Set version for local collection {} to {}", c.getName(), c.getZNodeVersion() + 1);
} catch (InterruptedException | AlreadyClosedException e) {
ParWork.propagateInterrupt(e);
throw e;
@@ -269,16 +269,6 @@ public class ZkStateWriter {
throw e;
} catch (Exception e) {
ParWork.propagateInterrupt(e);
-// if (e instanceof KeeperException.BadVersionException) {
-// // nocommit invalidState = true;
-// //if (log.isDebugEnabled())
-// log.info(
-// "Tried to update the cluster state using version={} but we where rejected, currently at {}",
-// prevVersion, c == null ? "null" : c.getZNodeVersion(), e);
-// prevState = reader.getClusterState();
-// continue;
-// }
- ParWork.propagateInterrupt(e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Failed processing update=" + c, e);
}
@@ -290,7 +280,10 @@ public class ZkStateWriter {
if (c != null && !failedUpdates.containsKey(c.getName())) {
try {
//System.out.println("waiting to see state " + prevVersion);
- Integer finalPrevVersion = c.getZNodeVersion();
+ Integer v = trackVersions.get(c.getName());
+ int version = v - 1;
+
+ Integer finalPrevVersion = version;
reader.waitForState(c.getName(), 15, TimeUnit.SECONDS, (l, col) -> {
// if (col != null) {
@@ -313,11 +306,11 @@ public class ZkStateWriter {
lastUpdatedTime = System.nanoTime();
- updatesToWrite.clear();
- if (log.isDebugEnabled()) log.debug("Failed updates {}", failedUpdates.values());
- // updatesToWrite.putAll(failedUpdates);
+
+
if (failedUpdates.size() > 0) {
- throw new AlreadyClosedException();
+ failedUpdates.clear();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, lastFailedException);
}
success = true;
@@ -334,8 +327,11 @@ public class ZkStateWriter {
// log.debug("writePendingUpdates() - end - New Cluster State is: {}", newClusterState);
// }
-
- return state;
+ try {
+ return new ClusterState(state.getLiveNodes(), updatesToWrite);
+ } finally {
+ updatesToWrite.clear();
+ }
}
public Map<String,DocCollection> getUpdatesToWrite() {
diff --git a/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java b/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java
index 7b0361d..30de73a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java
@@ -113,8 +113,6 @@ public class MissingSegmentRecoveryTest extends SolrCloudTestCase {
JettySolrRunner jetty = cluster.getReplicaJetty(replica);
jetty.stop();
jetty.start();
-
- cluster.waitForActiveCollection(collection, 1, 2);
QueryResponse resp = cluster.getSolrClient().query(collection, new SolrQuery("*:*"));
assertEquals(10, resp.getResults().getNumFound());
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 08b2590..6dc54c5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -852,16 +852,16 @@ public class ZkStateReader implements SolrCloseable {
notifications.shutdown();
}
+ waitLatches.forEach(c -> { for (int i = 0; i < c.getCount(); i++) c.countDown(); });
+
try {
if (closeClient) {
IOUtils.closeQuietly(zkClient);
}
- //waitLatches.forEach(c -> { for (int i = 0; i < c.getCount(); i++) c.countDown(); });
-
if (notifications != null) {
try {
- boolean success = notifications.awaitTermination(3, TimeUnit.SECONDS);
+ boolean success = notifications.awaitTermination(1, TimeUnit.SECONDS);
if (!success) notifications.shutdownNow();
} catch (InterruptedException e) {
ParWork.propagateInterrupt(e);
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java b/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java
index f15efc3..b903de0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java
@@ -904,7 +904,6 @@ public class SolrQueuedThreadPool extends ContainerLifeCycle implements ThreadFa
public void close() {
- this.closed = true;
removeBean(_tryExecutor);
_tryExecutor = TryExecutor.NO_TRY;
@@ -913,18 +912,25 @@ public class SolrQueuedThreadPool extends ContainerLifeCycle implements ThreadFa
//setIdleTimeout(1);
+ try {
+ super.doStop();
+ } catch (Exception e) {
+ LOG.warn("super.doStop", e);
+ return;
+ }
+
// Fill the job queue with noop jobs to wakeup idle threads.
for (int i = 0; i < threads; ++i) {
jobs.offer(NOOP);
}
// interrupt threads
- for (Future thread : _threadFutures)
- {
- if (LOG.isDebugEnabled())
- LOG.debug("Interrupting {}", thread);
- thread.cancel(true);
- }
+// for (Future thread : _threadFutures)
+// {
+// if (LOG.isDebugEnabled())
+// LOG.debug("Interrupting {}", thread);
+// thread.cancel(true);
+// }
// Close any un-executed jobs
@@ -947,14 +953,7 @@ public class SolrQueuedThreadPool extends ContainerLifeCycle implements ThreadFa
// }
- try {
- super.doStop();
- } catch (Exception e) {
- LOG.warn("super.doStop", e);
- return;
- }
-
-
+ this.closed = true;
try {
joinThreads(15000);
} catch (InterruptedException e) {