You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/03/09 23:27:29 UTC
[lucene-solr] 01/01: @1437 State publish cache back in.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit b662d81c4f4a04b423da7b469b9cca4d24e48c21
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Mar 9 17:21:59 2021 -0600
@1437 State publish cache back in.
Took 10 minutes
---
.../java/org/apache/solr/cloud/StatePublisher.java | 28 ++++++++++--------
.../java/org/apache/solr/cloud/ZkController.java | 2 ++
.../apache/solr/cloud/overseer/ZkStateWriter.java | 33 ++++++----------------
.../processor/DistributedZkUpdateProcessor.java | 2 +-
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 1 +
5 files changed, 30 insertions(+), 36 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java b/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
index b332fc8..880045f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
+++ b/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
@@ -22,6 +22,7 @@ import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
@@ -212,6 +213,11 @@ public class StatePublisher implements Closeable {
String collection = stateMessage.getStr(ZkStateReader.COLLECTION_PROP);
String state = stateMessage.getStr(ZkStateReader.STATE_PROP);
+ if (core == null || state == null) {
+ log.error("Nulls in published state");
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nulls in published state " + stateMessage);
+ }
+
if ((state.equals(UpdateLog.State.ACTIVE.toString().toLowerCase(Locale.ROOT)) || state.equals("leader")) && cc.isCoreLoading(core)) {
cc.waitForLoadingCore(core, 10000);
}
@@ -224,17 +230,13 @@ public class StatePublisher implements Closeable {
} else {
id = stateMessage.getStr("id");
}
- // MRM TODO: this needs thought and work - what about session recovery? what about zkshardterm recovery?
-// CacheEntry lastState = stateCache.get(id);
-// if (collection != null && replica != null && (System.currentTimeMillis() - lastState.time < 1000) && !state.equals(lastState.state) && replica.getState().toString().equals(state)) {
-// log.info("Skipping publish state as {} for {}, because it was the last state published", state, core);
-// return;
-// }
- }
- if (core == null || state == null) {
- log.error("Nulls in published state");
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nulls in published state " + stateMessage);
+ CacheEntry lastState = stateCache.get(id);
+ //&& (System.currentTimeMillis() - lastState.time < 1000) &&
+ if (!state.equals("leader") && collection != null && lastState != null && replica != null && !state.equals(lastState.state) && replica.getState().toString().equals(state)) {
+ log.info("Skipping publish state as {} for {}, because it was the last state published", state, core);
+ return;
+ }
}
if (id == null) {
@@ -297,7 +299,11 @@ public class StatePublisher implements Closeable {
}
public void clearStatCache(String core) {
- // stateCache.remove(core);
+ stateCache.remove(core);
+ }
+
+ public void clearStatCache() {
+ stateCache.clear();
}
public void start() {
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 9f69564..c205778 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -447,6 +447,8 @@ public class ZkController implements Closeable, Runnable {
removeEphemeralLiveNode();
+ statePublisher.clearStatCache();
+
// recreate our watchers first so that they exist even on any problems below
zkStateReader.createClusterStateWatchersAndUpdate();
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 20c4f55..06675de 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -112,35 +112,22 @@ public class ZkStateWriter {
throw new NullPointerException("clusterState cannot be null");
}
-
clusterState.forEachCollection(collection -> {
- idToCollection.put(collection.getId(), collection.getName());
-// if (trackVersions.get(collection.getName()) == null) {
-// DocCollection latestColl = reader.getClusterState().getCollectionOrNull(collection.getName()).copy();
-//
-// if (latestColl == null) {
-// reader.forciblyRefreshClusterStateSlow(collection.getName());
-// latestColl = reader.getClusterState().getCollectionOrNull(collection.getName()).copy();
-// }
-//
-// if (latestColl == null) {
-// //log.info("no node exists, using version 0");
-// trackVersions.remove(collection.getName());
-// } else {
-// int version = latestColl.getZNodeVersion();
-//
-// log.debug("Updating local tracked version to {} for {}", version, collection.getName());
-// trackVersions.put(collection.getName(), version);
-// }
-// }
-
DocCollection currentCollection = cs.getCollectionOrNull(collection.getName());
log.debug("zkwriter collection={}", collection);
log.debug("zkwriter currentCollection={}", currentCollection);
+ idToCollection.put(collection.getId(), collection.getName());
+
+ if (currentCollection != null) {
+ if (currentCollection.getId() != collection.getId()) {
+ removeCollection(collection.getName());
+ }
+ }
+
+ if (currentCollection != null) {
- if (currentCollection != null && currentCollection.getId() == collection.getId()) {
currentCollection.getProperties().keySet().retainAll(collection.getProperties().keySet());
for (Slice slice : collection) {
@@ -612,8 +599,6 @@ public class ZkStateWriter {
cs.getCollectionStates().remove(collection);
assignMap.remove(collection);
trackVersions.remove(collection);
- reader.getZkClient().delete(ZkStateReader.getCollectionSCNPath(collection), -1, true, false);
- reader.getZkClient().delete(ZkStateReader.getCollectionStateUpdatesPath(collection), -1, true, false);
dirtyStructure.remove(collection);
dirtyState.remove(collection);
ZkNodeProps message = new ZkNodeProps("name", collection);
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index 2a7701d..b5ce87c 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -524,7 +524,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
String sliceName = slice.getName();
Replica leader;
try {
- leader = zkController.getZkStateReader().getLeaderRetry(collection, sliceName);
+ leader = zkController.getZkStateReader().getLeaderRetry(collection, sliceName, 10000);
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "error getting leader", e);
}
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index d11471c..228b60b 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -2314,6 +2314,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
return new ConcurrentUpdateSolrClient.Builder(baseSolrUrl)
.withQueueSize(queueSize)
.withThreadCount(threadCount)
+ .withSocketTimeout(120000)
.build();
}