You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2023/10/30 06:08:41 UTC
(lucene-solr) branch branch_8_11 updated (002b3ce67be -> a442489af3c)
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a change to branch branch_8_11
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
from 002b3ce67be Revert "SOLR-15694, 15715: Create ephemeral roles properly (missed backporting these changes last time)"
new ace7641edb7 SOLR-15694, 15715: Create ephemeral roles properly (missed backporting these changes last time)
new a442489af3c Some PRS fixes ported (addressing test failures with PRS)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/solr/cloud/ZkController.java | 32 +++++++++++++++++++++-
.../OverseerCollectionMessageHandler.java | 2 +-
.../autoscaling/sim/SimClusterStateProvider.java | 5 +++-
.../solr/handler/admin/CollectionsHandler.java | 3 +-
.../apache/solr/search/TestCoordinatorRole.java | 4 ++-
.../apache/solr/common/cloud/PerReplicaStates.java | 21 +++++++++++++-
6 files changed, 61 insertions(+), 6 deletions(-)
(lucene-solr) 01/02: SOLR-15694, 15715: Create ephemeral roles properly (missed backporting these changes last time)
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch branch_8_11
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit ace7641edb762d31519df1db845885dfd31caee4
Author: noblepaul <no...@gmail.com>
AuthorDate: Mon Oct 30 05:56:24 2023 +1100
SOLR-15694, 15715: Create ephemeral roles properly (missed backporting these changes last time)
---
.../java/org/apache/solr/cloud/ZkController.java | 32 +++++++++++++++++++++-
.../apache/solr/search/TestCoordinatorRole.java | 4 ++-
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 6f6bcbf24dc..56bd5eab66c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -106,6 +106,7 @@ import org.apache.solr.core.CloseHook;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.NodeRoles;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrCoreInitializationException;
import org.apache.solr.handler.admin.ConfigSetsHandler;
@@ -700,7 +701,7 @@ public class ZkController implements Closeable {
// if this replica is not a leader, it will be put in recovery state by the leader
String leader = cd.getCloudDescriptor().getCoreNodeName();
- if (shard.getReplica(leader) != shard.getLeader()) return;
+ if (!Objects.equals(shard.getReplica(leader), shard.getLeader())) return;
Set<String> liveNodes = getClusterState().getLiveNodes();
int numActiveReplicas = shard.getReplicas(
@@ -858,6 +859,14 @@ public class ZkController implements Closeable {
throws KeeperException, InterruptedException, IOException {
ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient);
+ cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES, zkClient);
+ for (NodeRoles.Role role : NodeRoles.Role.values()) {
+ cmdExecutor.ensureExists(NodeRoles.getZNodeForRole(role), zkClient);
+ for (String mode : role.supportedModes()) {
+ cmdExecutor.ensureExists(NodeRoles.getZNodeForRoleMode(role, mode), zkClient);
+ }
+ }
+
cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient);
cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient);
cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH, zkClient);
@@ -1171,6 +1180,27 @@ public class ZkController implements Closeable {
ops.add(Op.create(nodeAddedPath, json, zkClient.getZkACLProvider().getACLsToAdd(nodeAddedPath), CreateMode.EPHEMERAL));
}
zkClient.multi(ops, true);
+ Map<NodeRoles.Role, String> roles = cc.nodeRoles.getRoles();
+
+ try {
+ zkClient.create(nodePath, null, CreateMode.EPHEMERAL, true);
+ } catch (KeeperException.NodeExistsException e) {
+ // ignore , it's already there
+ }
+
+ for (Map.Entry<NodeRoles.Role, String> e : roles.entrySet()) {
+ try {
+ zkClient.create(
+ NodeRoles.getZNodeForRoleMode(e.getKey(), e.getValue()) + "/" + nodeName,
+ null,
+ CreateMode.EPHEMERAL,
+ true);
+ } catch (KeeperException.NodeExistsException ex) {
+ // ignore , it already exists
+ }
+ }
+
+ log.info("non-data nodes now {}", zkClient.getChildren("/node_roles/data/off", null,true));
}
public void removeEphemeralLiveNode() throws KeeperException, InterruptedException {
diff --git a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java
index 430efe573e1..14239dac26f 100644
--- a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java
+++ b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java
@@ -688,7 +688,9 @@ public class TestCoordinatorRole extends SolrCloudTestCase {
assertTrue(zkWatchAccessor.getWatchedCollections().contains(TEST_COLLECTION_1));
// add another collection
- CollectionAdminRequest.createCollection(TEST_COLLECTION_2, "conf1", 1, 2).process(client);
+ CollectionAdminRequest.createCollection(TEST_COLLECTION_2, "conf1", 1, 2)
+ .setMaxShardsPerNode(10)
+ .process(client);
cluster.waitForActiveCollection(TEST_COLLECTION_2, 1, 2);
new QueryRequest(new SolrQuery("*:*"))
.setPreferredNodes(ImmutableList.of(coordinatorJetty.getNodeName()))
(lucene-solr) 02/02: Some PRS fixes ported (addressing test failures with PRS)
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch branch_8_11
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit a442489af3c665e1d87c38bef1b5864dd48129a7
Author: Ishan Chattopadhyaya <is...@searchscale.com>
AuthorDate: Mon Oct 30 11:25:39 2023 +0530
Some PRS fixes ported (addressing test failures with PRS)
---
.../OverseerCollectionMessageHandler.java | 2 +-
.../autoscaling/sim/SimClusterStateProvider.java | 5 ++++-
.../solr/handler/admin/CollectionsHandler.java | 3 ++-
.../apache/solr/common/cloud/PerReplicaStates.java | 21 ++++++++++++++++++++-
4 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index b0155979650..eea3ff7c50e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -997,7 +997,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
Slice slice, ShardHandler shardHandler) {
List<Replica> notLiveReplicas = new ArrayList<>();
for (Replica replica : slice.getReplicas()) {
- if ((stateMatcher == null || Replica.State.getState(replica.getStr(ZkStateReader.STATE_PROP)) == stateMatcher)) {
+ if ((stateMatcher == null || replica.getState() == stateMatcher)) {
if (clusterState.liveNodesContain(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
// For thread safety, only simple clone the ModifiableSolrParams
ModifiableSolrParams cloneParams = new ModifiableSolrParams();
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 13ad560aa4b..0079eecf7c1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -45,6 +45,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
+import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.AtomicDouble;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
@@ -78,6 +79,7 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.cloud.Slice;
@@ -2461,7 +2463,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
@SuppressWarnings({"unchecked"})
Map<String, Object> routerProp = (Map<String, Object>) collProps.getOrDefault(DocCollection.DOC_ROUTER, Collections.singletonMap("name", DocRouter.DEFAULT_NAME));
DocRouter router = DocRouter.getDocRouter((String)routerProp.getOrDefault("name", DocRouter.DEFAULT_NAME));
- DocCollection dc = new DocCollection(coll, slices, collProps, router, clusterStateVersion, ZkStateReader.CLUSTER_STATE);
+ DocCollection dc = new DocCollection(coll, slices, collProps, router, clusterStateVersion, ZkStateReader.CLUSTER_STATE,
+ new DocCollection.PrsSupplier(new PerReplicaStates(ZkStateReader.getCollectionPath(coll), 0, ImmutableList.of())));
res.put(coll, dc);
});
saveClusterState(new ClusterState(clusterStateVersion, liveNodes.get(), res));
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 0eec0789aad..d933a776cc6 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -1014,7 +1014,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
Map<String, Object> all = copy(req.getParams(), null,
COLLECTION_PROP,
SHARD_ID_PROP,
- _ROUTE_);
+ _ROUTE_,
+ "prs");
new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(),
new ZkNodeProps(all)).getClusterStatus(rsp.getValues());
return null;
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/PerReplicaStates.java b/solr/solrj/src/java/org/apache/solr/common/cloud/PerReplicaStates.java
index 9581468017f..b025ab1b3e2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/PerReplicaStates.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/PerReplicaStates.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;
import org.apache.solr.cluster.api.SimpleMap;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
@@ -326,10 +327,28 @@ public class PerReplicaStates implements ReflectMapWriter {
return duplicate;
}
}
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ReflectMapWriter.super.writeMap(
+ new EntryWriter() {
+ @Override
+ public EntryWriter put(CharSequence k, Object v) throws IOException {
+ if ("states".equals(k.toString())) {
+ ew.put(
+ "states",
+ (IteratorWriter)
+ iw -> states.forEachEntry((s, state) -> iw.addNoEx(state.toString())));
+ } else {
+ ew.put(k, v);
+ }
+ return this;
+ }
+ });
+ }
public static class LazyPrsSupplier extends DocCollection.PrsSupplier {
public LazyPrsSupplier(SolrZkClient zkClient, String collectionPath) {
super(() -> fetch(collectionPath, zkClient, null));
}
}
-
}