You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/03/10 14:18:21 UTC
[lucene-solr] branch reference_impl updated: @1443 Make some more
real world tweaks, allow config of per thread exec.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/reference_impl by this push:
new c8f7f3c @1443 Make some more real world tweaks, allow config of per thread exec.
c8f7f3c is described below
commit c8f7f3c5f2255a79dcf5cb748ef77308bd2d64b4
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Mar 10 08:17:48 2021 -0600
@1443 Make some more real world tweaks, allow config of per thread exec.
Took 5 minutes
---
.../java/org/apache/solr/cloud/StatePublisher.java | 6 +--
.../cloud/api/collections/CreateCollectionCmd.java | 2 +-
.../apache/solr/cloud/overseer/ZkStateWriter.java | 48 ++++++++++------------
.../java/org/apache/solr/servlet/HttpSolrCall.java | 2 +-
.../src/java/org/apache/solr/common/ParWork.java | 2 +-
.../apache/solr/common/util/ValidatingJsonMap.java | 4 +-
6 files changed, 30 insertions(+), 34 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java b/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
index 880045f..49ad9e9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
+++ b/solr/core/src/java/org/apache/solr/cloud/StatePublisher.java
@@ -218,9 +218,9 @@ public class StatePublisher implements Closeable {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nulls in published state " + stateMessage);
}
- if ((state.equals(UpdateLog.State.ACTIVE.toString().toLowerCase(Locale.ROOT)) || state.equals("leader")) && cc.isCoreLoading(core)) {
- cc.waitForLoadingCore(core, 10000);
- }
+// if ((state.equals(UpdateLog.State.ACTIVE.toString().toLowerCase(Locale.ROOT)) || state.equals("leader")) && cc.isCoreLoading(core)) {
+// cc.waitForLoadingCore(core, 10000);
+// }
DocCollection coll = zkStateReader.getClusterState().getCollectionOrNull(collection);
if (coll != null) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 70c99de..8e6f01f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -251,7 +251,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
List<Replica> replicas = wcollection.getReplicas(nodeName);
if (replicas == null || replicas.isEmpty()) {
ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toString(), ZkStateReader.COLLECTION_PROP, withCollection, ZkStateReader.SHARD_ID_PROP, withCollectionShard,
- "node", nodeName, CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created
+ "node", nodeName, ZkStateReader.NODE_NAME_PROP, nodeName, CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created
new AddReplicaCmd(ocmh, true).call(clusterState, props, results);
clusterState = new SliceMutator(cloudManager).addReplica(clusterState, props, ocmh.overseer);
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 9f91b41..e406904 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -100,8 +100,6 @@ public class ZkStateWriter {
}
public void enqueueUpdate(ClusterState clusterState, ZkNodeProps message, boolean stateUpdate) throws Exception {
-
-
//log.info("Get our write lock for enq");
ourLock.lock();
//log.info("Got our write lock for enq");
@@ -344,26 +342,24 @@ public class ZkStateWriter {
log.debug("zkwriter set node operation {} for {} cs={}}", operation, entry.getValue(), cs);
ClusterState clusterState = cs;
-//
-// if (cs.getCollectionStates().size() == 0) {
-// reader.forciblyRefreshAllClusterStateSlow();
-// clusterState = reader.getClusterState().copy();
-// log.debug("set operation try again {} for {} cs={}}", operation, entry.getValue(), clusterState);
-// // cs = clusterState;
-// }
+ //
+ // if (cs.getCollectionStates().size() == 0) {
+ // reader.forciblyRefreshAllClusterStateSlow();
+ // clusterState = reader.getClusterState().copy();
+ // log.debug("set operation try again {} for {} cs={}}", operation, entry.getValue(), clusterState);
+ // // cs = clusterState;
+ // }
clusterState.forEachCollection(docColl -> {
if (trackVersions.get(docColl.getName()) == null) {
- // reader.forciblyRefreshClusterStateSlow(docColl.getName());
-
+ // reader.forciblyRefreshClusterStateSlow(docColl.getName());
-
- //log.info("got version from zk {}", existsStat.getVersion());
- int version = docColl.getZNodeVersion();
- log.info("Updating local tracked version to {} for {}", version, docColl.getName());
- trackVersions.put(docColl.getName(), version);
- idToCollection.put(docColl.getId(), docColl.getName());
+ //log.info("got version from zk {}", existsStat.getVersion());
+ int version = docColl.getZNodeVersion();
+ log.info("Updating local tracked version to {} for {}", version, docColl.getName());
+ trackVersions.put(docColl.getName(), version);
+ idToCollection.put(docColl.getId(), docColl.getName());
}
@@ -377,20 +373,20 @@ public class ZkStateWriter {
ver = 1;
}
updates.getProperties().put("_cs_ver_", ver.toString());
- // dirtyState.add(docColl.getName());
- // dirtyStructure.add(docColl.getName());
+ // dirtyState.add(docColl.getName());
+ // dirtyStructure.add(docColl.getName());
List<Replica> replicas = docColl.getReplicas();
log.debug("update replicas with node operation {} reps={}", operation, replicas.size());
for (Replica replica : replicas) {
if (!Replica.State.getShortState(replica.getState()).equals(operation) && replica.getNodeName().equals(entry.getValue())) {
if (log.isDebugEnabled()) log.debug("set node operation {} for replica {}", operation, replica);
// MRM TODO:
- // Slice slice = docColl.getSlice(replica.getSlice());
-// Replica leaderReplica = slice.getLeader();
-// if (leaderReplica != null && replica == leaderReplica) {
-// leaderReplica.getProperties().remove("leader");
-// slice.setLeader(null);
-// }
+ // Slice slice = docColl.getSlice(replica.getSlice());
+ // Replica leaderReplica = slice.getLeader();
+ // if (leaderReplica != null && replica == leaderReplica) {
+ // leaderReplica.getProperties().remove("leader");
+ // slice.setLeader(null);
+ // }
replica.setState(Replica.State.shortStateToState(operation));
updates.getProperties().put(replica.getId(), operation);
dirtyState.add(docColl.getName());
@@ -573,13 +569,13 @@ public class ZkStateWriter {
private void writeStateUpdates(DocCollection collection, ZkNodeProps updates) throws KeeperException, InterruptedException {
String stateUpdatesPath = ZkStateReader.getCollectionStateUpdatesPath(collection.getName());
if (log.isDebugEnabled()) log.debug("write state updates for collection {} {}", collection.getName(), updates);
- dirtyState.remove(collection.getName());
try {
reader.getZkClient().setData(stateUpdatesPath, Utils.toJSON(updates), -1, true, false);
} catch (KeeperException.NoNodeException e) {
if (log.isDebugEnabled()) log.debug("No node found for state.json", e);
// likely deleted
}
+ dirtyState.remove(collection.getName());
}
public ClusterState getClusterstate() {
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 11692a5..b95256d2 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -260,7 +260,7 @@ public class HttpSolrCall {
core = cores.getCore(origCorename);
if (core == null && cores.isCoreLoading(origCorename)) {
- cores.waitForLoadingCore(origCorename, 10000);
+ cores.waitForLoadingCore(origCorename, 5000);
core = cores.getCore(origCorename);
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWork.java b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
index 4e2201c..898fb1c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWork.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
@@ -435,7 +435,7 @@ public class ParWork implements Closeable {
Integer minThreads;
Integer maxThreads;
minThreads = 4;
- maxThreads = PROC_COUNT;
+ maxThreads = Integer.getInteger("solr.perThreadPoolSize", PROC_COUNT / 2);
exec = getExecutorService(Math.max(minThreads, maxThreads)); // keep alive directly affects how long a worker might
// ((PerThreadExecService)exec).closeLock(true);
// be stuck in poll without an enqueue on shutdown
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ValidatingJsonMap.java b/solr/solrj/src/java/org/apache/solr/common/util/ValidatingJsonMap.java
index 32a5b81..7717d3d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ValidatingJsonMap.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ValidatingJsonMap.java
@@ -268,7 +268,7 @@ public class ValidatingJsonMap implements Map<String, Object>, NavigableObject {
handleIncludes(map, includeLocation, 4);
return map;
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
@@ -357,7 +357,7 @@ public class ValidatingJsonMap implements Map<String, Object>, NavigableObject {
return getDeepCopy(map, 5, false);
}
} catch (IOException e) {
-
+ // TODO: at least log
}
}
}