You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/03/09 23:27:28 UTC
[lucene-solr] branch reference_impl_dev updated (35104f0 -> b662d81)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 35104f0 @1433 Work out a silly little bug that sent me all over creation.
add 075f034 @1434 Couple tweaks around state updates.
add a7ab3bd @1435 Debug logging off.
add fd720f8 @1436 Tweaks and put in the wrong node error again.
new b662d81 @1437 State publish cache back in.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/solr/cloud/StatePublisher.java | 28 ++++++----
.../java/org/apache/solr/cloud/ZkController.java | 18 ++++---
.../OverseerCollectionMessageHandler.java | 3 +-
.../apache/solr/cloud/overseer/ZkStateWriter.java | 61 ++++++----------------
.../processor/DistributedZkUpdateProcessor.java | 2 +-
solr/server/resources/log4j2.xml | 44 ++++------------
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 1 +
7 files changed, 56 insertions(+), 101 deletions(-)
[lucene-solr] 01/01: @1437 State publish cache back in.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit b662d81c4f4a04b423da7b469b9cca4d24e48c21
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Mar 9 17:21:59 2021 -0600
@1437 State publish cache back in.
Took 10 minutes
---
.../java/org/apache/solr/cloud/StatePublisher.java | 28 ++++++++++--------
.../java/org/apache/solr/cloud/ZkController.java | 2 ++
.../apache/solr/cloud/overseer/ZkStateWriter.java | 33 ++++++----------------
.../processor/DistributedZkUpdateProcessor.java | 2 +-
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 1 +
5 files changed, 30 insertions(+), 36 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java b/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
index b332fc8..880045f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
+++ b/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
@@ -22,6 +22,7 @@ import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
@@ -212,6 +213,11 @@ public class StatePublisher implements Closeable {
String collection = stateMessage.getStr(ZkStateReader.COLLECTION_PROP);
String state = stateMessage.getStr(ZkStateReader.STATE_PROP);
+ if (core == null || state == null) {
+ log.error("Nulls in published state");
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nulls in published state " + stateMessage);
+ }
+
if ((state.equals(UpdateLog.State.ACTIVE.toString().toLowerCase(Locale.ROOT)) || state.equals("leader")) && cc.isCoreLoading(core)) {
cc.waitForLoadingCore(core, 10000);
}
@@ -224,17 +230,13 @@ public class StatePublisher implements Closeable {
} else {
id = stateMessage.getStr("id");
}
- // MRM TODO: this needs thought and work - what about session recovery? what about zkshardterm recovery?
-// CacheEntry lastState = stateCache.get(id);
-// if (collection != null && replica != null && (System.currentTimeMillis() - lastState.time < 1000) && !state.equals(lastState.state) && replica.getState().toString().equals(state)) {
-// log.info("Skipping publish state as {} for {}, because it was the last state published", state, core);
-// return;
-// }
- }
- if (core == null || state == null) {
- log.error("Nulls in published state");
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nulls in published state " + stateMessage);
+ CacheEntry lastState = stateCache.get(id);
+ //&& (System.currentTimeMillis() - lastState.time < 1000) &&
+ if (!state.equals("leader") && collection != null && lastState != null && replica != null && !state.equals(lastState.state) && replica.getState().toString().equals(state)) {
+ log.info("Skipping publish state as {} for {}, because it was the last state published", state, core);
+ return;
+ }
}
if (id == null) {
@@ -297,7 +299,11 @@ public class StatePublisher implements Closeable {
}
public void clearStatCache(String core) {
- // stateCache.remove(core);
+ stateCache.remove(core);
+ }
+
+ public void clearStatCache() {
+ stateCache.clear();
}
public void start() {
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 9f69564..c205778 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -447,6 +447,8 @@ public class ZkController implements Closeable, Runnable {
removeEphemeralLiveNode();
+ statePublisher.clearStatCache();
+
// recreate our watchers first so that they exist even on any problems below
zkStateReader.createClusterStateWatchersAndUpdate();
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 20c4f55..06675de 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -112,35 +112,22 @@ public class ZkStateWriter {
throw new NullPointerException("clusterState cannot be null");
}
-
clusterState.forEachCollection(collection -> {
- idToCollection.put(collection.getId(), collection.getName());
-// if (trackVersions.get(collection.getName()) == null) {
-// DocCollection latestColl = reader.getClusterState().getCollectionOrNull(collection.getName()).copy();
-//
-// if (latestColl == null) {
-// reader.forciblyRefreshClusterStateSlow(collection.getName());
-// latestColl = reader.getClusterState().getCollectionOrNull(collection.getName()).copy();
-// }
-//
-// if (latestColl == null) {
-// //log.info("no node exists, using version 0");
-// trackVersions.remove(collection.getName());
-// } else {
-// int version = latestColl.getZNodeVersion();
-//
-// log.debug("Updating local tracked version to {} for {}", version, collection.getName());
-// trackVersions.put(collection.getName(), version);
-// }
-// }
-
DocCollection currentCollection = cs.getCollectionOrNull(collection.getName());
log.debug("zkwriter collection={}", collection);
log.debug("zkwriter currentCollection={}", currentCollection);
+ idToCollection.put(collection.getId(), collection.getName());
+
+ if (currentCollection != null) {
+ if (currentCollection.getId() != collection.getId()) {
+ removeCollection(collection.getName());
+ }
+ }
+
+ if (currentCollection != null) {
- if (currentCollection != null && currentCollection.getId() == collection.getId()) {
currentCollection.getProperties().keySet().retainAll(collection.getProperties().keySet());
for (Slice slice : collection) {
@@ -612,8 +599,6 @@ public class ZkStateWriter {
cs.getCollectionStates().remove(collection);
assignMap.remove(collection);
trackVersions.remove(collection);
- reader.getZkClient().delete(ZkStateReader.getCollectionSCNPath(collection), -1, true, false);
- reader.getZkClient().delete(ZkStateReader.getCollectionStateUpdatesPath(collection), -1, true, false);
dirtyStructure.remove(collection);
dirtyState.remove(collection);
ZkNodeProps message = new ZkNodeProps("name", collection);
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index 2a7701d..b5ce87c 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -524,7 +524,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
String sliceName = slice.getName();
Replica leader;
try {
- leader = zkController.getZkStateReader().getLeaderRetry(collection, sliceName);
+ leader = zkController.getZkStateReader().getLeaderRetry(collection, sliceName, 10000);
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "error getting leader", e);
}
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index d11471c..228b60b 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -2314,6 +2314,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
return new ConcurrentUpdateSolrClient.Builder(baseSolrUrl)
.withQueueSize(queueSize)
.withThreadCount(threadCount)
+ .withSocketTimeout(120000)
.build();
}