You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2021/01/18 12:47:24 UTC
[lucene-solr] branch jira/solr_13951 updated (290ea6a -> f57ea28)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a change to branch jira/solr_13951
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 290ea6a merging with 8x
new ff69c0c merging with 8x
new f57ea28 added CHANGES.txt extry
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
solr/CHANGES.txt | 2 ++
.../java/org/apache/solr/cloud/ZkController.java | 1 +
.../apache/solr/cloud/overseer/ZkStateWriter.java | 6 ++----
.../solr/handler/admin/CollectionsHandler.java | 13 ++----------
.../client/solrj/cloud/DistribStateManager.java | 7 +++++++
.../client/solrj/impl/ZkDistribStateManager.java | 6 ++++++
.../org/apache/solr/common/cloud/ClusterState.java | 8 +++++---
.../java/org/apache/solr/common/cloud/Replica.java | 23 +++++++++++++---------
.../client/solrj/impl/CloudSolrClientTest.java | 2 +-
9 files changed, 40 insertions(+), 28 deletions(-)
[lucene-solr] 02/02: added CHANGES.txt extry
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr_13951
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit f57ea2802a0b78c986533f1cafcb9c0d97872264
Author: noblepaul <no...@gmail.com>
AuthorDate: Mon Jan 18 23:46:59 2021 +1100
added CHANGES.txt extry
---
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c704930..dcdc005 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -203,6 +203,8 @@ New Features
* SOLR-14560: Add interleaving support in Learning To Rank. (Alessandro Benedetti, Christine Poerschke)
+* SOLR-15052: Reducing overseer bottlenecks using per-replica states (noble, Ishan Chattopadhyaya)
+
Improvements
---------------------
* SOLR-14942: Reduce leader election time on node shutdown by removing election nodes before closing cores.
[lucene-solr] 01/02: merging with 8x
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr_13951
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ff69c0c46c1eb476e6037894e2856ebb6770a574
Author: noblepaul <no...@gmail.com>
AuthorDate: Mon Jan 18 17:00:48 2021 +1100
merging with 8x
---
.../java/org/apache/solr/cloud/ZkController.java | 1 +
.../apache/solr/cloud/overseer/ZkStateWriter.java | 6 ++----
.../solr/handler/admin/CollectionsHandler.java | 13 ++----------
.../client/solrj/cloud/DistribStateManager.java | 7 +++++++
.../client/solrj/impl/ZkDistribStateManager.java | 6 ++++++
.../org/apache/solr/common/cloud/ClusterState.java | 8 +++++---
.../java/org/apache/solr/common/cloud/Replica.java | 23 +++++++++++++---------
.../client/solrj/impl/CloudSolrClientTest.java | 2 +-
8 files changed, 38 insertions(+), 28 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index ee4da55..dd55332 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1610,6 +1610,7 @@ public class ZkController implements Closeable {
*/
static boolean sendToOverseer(DocCollection coll, String replicaName) {
if (coll == null) return true;
+ if (!coll.isPerReplicaState()) return true;
Replica r = coll.getReplica(replicaName);
if (r == null) return true;
Slice shard = coll.getSlice(r.shard);
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 1ebccd2..2ff739b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -148,10 +148,8 @@ public class ZkStateWriter {
for (ZkWriteCommand cmd : cmds) {
if (cmd == NO_OP) continue;
prevState = prevState.copyWith(cmd.name, cmd.collection);
- if (cmd.collection == null) {
- updates.put(cmd.name, cmd);
- numUpdates++;
- }
+ updates.put(cmd.name, cmd);
+ numUpdates++;
}
clusterState = prevState;
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 974699c..5056b44 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -19,17 +19,8 @@ package org.apache.solr.handler.admin;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java
index cac6cd2..feae27f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.NoSuchElementException;
import org.apache.solr.common.SolrCloseable;
+import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Op;
@@ -100,6 +101,12 @@ public interface DistribStateManager extends SolrCloseable {
return tree;
}
+ default PerReplicaStates getReplicaStates(String path) throws KeeperException, InterruptedException {
+ throw new UnsupportedOperationException("Not implemented");
+
+
+ }
+
/**
* Remove data recursively.
* @param root root path
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java
index d2c3f40..fe4b961 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java
@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.NotEmptyException;
import org.apache.solr.client.solrj.cloud.VersionedData;
import org.apache.solr.common.AlreadyClosedException;
+import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -181,4 +182,9 @@ public class ZkDistribStateManager implements DistribStateManager {
public SolrZkClient getZkClient() {
return zkClient;
}
+
+ @Override
+ public PerReplicaStates getReplicaStates(String path) throws KeeperException, InterruptedException {
+ return PerReplicaStates.fetch(path, zkClient, null);
+ }
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index 5f8ee6b..3f05cee 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -248,8 +248,10 @@ public class ClusterState implements JSONWriter.Writable {
Map<String,Object> props;
Map<String,Slice> slices;
- if("true".equals(String.valueOf(objs.get(DocCollection.PER_REPLICA_STATE)))) {
- log.info("a collection {} has per-replica state" , name);
+ if (Boolean.parseBoolean(String.valueOf(objs.get(DocCollection.PER_REPLICA_STATE)))) {
+ if(log.isDebugEnabled()) {
+ log.debug("a collection {} has per-replica state", name);
+ }
//this collection has replica states stored outside
ReplicaStatesProvider rsp = REPLICASTATES_PROVIDER.get();
if (rsp instanceof StatesProvider) ((StatesProvider) rsp).isPerReplicaState = true;
@@ -448,7 +450,7 @@ public class ClusterState implements JSONWriter.Writable {
@Override
public PerReplicaStates getStates() {
- if(perReplicaStates == null) perReplicaStates = replicaStatesSupplier.get();
+ if (perReplicaStates == null) perReplicaStates = replicaStatesSupplier.get();
return perReplicaStates;
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
index f3253b6..27734db 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
@@ -151,6 +151,7 @@ public class Replica extends ZkNodeProps implements MapWriter {
this.node = (String) propMap.get(ZkStateReader.NODE_NAME_PROP);
this.core = (String) propMap.get(ZkStateReader.CORE_NAME_PROP);
this.type = Type.get((String) propMap.get(ZkStateReader.REPLICA_TYPE));
+ readPrs();
// default to ACTIVE
this.state = State.getState(String.valueOf(propMap.getOrDefault(ZkStateReader.STATE_PROP, State.ACTIVE.toString())));
validate();
@@ -172,6 +173,7 @@ public class Replica extends ZkNodeProps implements MapWriter {
if (props != null) {
this.propMap.putAll(props);
}
+ readPrs();
validate();
propMap.put(BASE_URL_PROP, UrlScheme.INSTANCE.getBaseUrlForNodeName(this.node));
}
@@ -193,6 +195,14 @@ public class Replica extends ZkNodeProps implements MapWriter {
this.node = String.valueOf(details.get("node_name"));
this.propMap.putAll(details);
+ readPrs();
+ type = Replica.Type.valueOf(String.valueOf(propMap.getOrDefault(ZkStateReader.REPLICA_TYPE, "NRT")));
+ if(state == null) state = State.getState(String.valueOf(propMap.getOrDefault(ZkStateReader.STATE_PROP, "active")));
+ validate();
+ propMap.put(BASE_URL_PROP, UrlScheme.INSTANCE.getBaseUrlForNodeName(this.node));
+ }
+
+ private void readPrs() {
ClusterState.getReplicaStatesProvider().get().ifPresent(it -> {
log.debug("A replica {} state fetched from per-replica state", name);
replicaState = it.getStates().get(name);
@@ -201,10 +211,6 @@ public class Replica extends ZkNodeProps implements MapWriter {
if (replicaState.isLeader) propMap.put(Slice.LEADER, "true");
}
}) ;
- type = Replica.Type.valueOf(String.valueOf(propMap.getOrDefault(ZkStateReader.REPLICA_TYPE, "NRT")));
- if(state == null) state = State.getState(String.valueOf(propMap.getOrDefault(ZkStateReader.STATE_PROP, "active")));
- validate();
- propMap.put(BASE_URL_PROP, UrlScheme.INSTANCE.getBaseUrlForNodeName(this.node));
}
private final void validate() {
@@ -295,11 +301,6 @@ public class Replica extends ZkNodeProps implements MapWriter {
return this.type;
}
- public boolean isLeader() {
- if(replicaState != null) return replicaState.isLeader;
- return getBool(ZkStateReader.LEADER_PROP, false);
- }
-
public Object get(String key, Object defValue) {
Object o = get(key);
if (o != null) {
@@ -383,6 +384,10 @@ public class Replica extends ZkNodeProps implements MapWriter {
.put(ZkStateReader.STATE_PROP, state.toString(), p);
};
}
+ public boolean isLeader() {
+ if (replicaState != null) return replicaState.isLeader;
+ return getStr(Slice.LEADER) != null;
+ }
@Override
public void write(JSONWriter jsonWriter) {
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
index f655b21..e156607 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
@@ -1076,7 +1076,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
testCollection = "perReplicaState_testv2";
new V2Request.Builder("/collections")
.withMethod(POST)
- .withPayload("{create: {name: perReplicaState_testv2, config : conf, numShards : 2, nrtReplicas : 2, perReplicaState : true, maxShardsPerNode : 5}}")
+ .withPayload("{create: {name: perReplicaState_testv2, config : conf, numShards : 2, nrtReplicas : 2, perReplicaState : true}}")
.build()
.process(cluster.getSolrClient());
cluster.waitForActiveCollection(testCollection, 2, 4);