You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/11/11 23:21:17 UTC
[lucene-solr] 03/03: @1170 Only dirty the collections we change.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit dbc91c2ee62506fb6a4c7c1f7010a5c21afa0413
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Nov 11 17:07:43 2020 -0600
@1170 Only dirty the collections we change.
---
.../solr/cloud/OverseerTaskExecutorTask.java | 3 --
.../cloud/api/collections/CreateCollectionCmd.java | 3 +-
.../solr/cloud/api/collections/MoveReplicaCmd.java | 14 ++++++++-
.../OverseerCollectionMessageHandler.java | 35 +++++++++++++++++-----
.../solr/cloud/api/collections/SplitShardCmd.java | 12 +++++++-
.../apache/solr/cloud/overseer/ZkStateWriter.java | 9 +-----
6 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java
index 2277034..52840ca 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java
@@ -70,10 +70,7 @@ public class OverseerTaskExecutorTask implements Runnable {
log.info("Queue operation is {}", operation);
- ClusterState cs = zkStateWriter.getClusterstate(true);
-
log.info("Process message {} {}", message, operation);
- // ClusterState newClusterState = processMessage(message, operation, cs);
log.info("Enqueue message {}", operation);
zkStateWriter.enqueueUpdate(null, message, true);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 9e29e30..4aaaee8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -331,7 +331,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
log.error("Exception trying to clean up collection after fail {}", collectionName);
}
if (log.isDebugEnabled()) log.debug("Cleaned up artifacts for failed create collection for [{}]", collectionName);
- //throw new SolrException(ErrorCode.BAD_REQUEST, "Underlying core creation failed while creating collection: " + collectionName + "\n" + results);
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Underlying core creation failed while creating collection: " + collectionName + "\n" + results);
} else {
Object createNodeSet = message.get(ZkStateReader.CREATE_NODE_SET);
if (log.isDebugEnabled()) log.debug("createNodeSet={}", createNodeSet);
@@ -379,6 +379,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
};
}
+ if (log.isDebugEnabled()) log.debug("return cs from create collection cmd {}", clusterState);
response.clusterState = clusterState;
return response;
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
index 691e267..a9c5a0b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
@@ -20,8 +20,10 @@ package org.apache.solr.cloud.api.collections;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -49,6 +51,7 @@ import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHan
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
@@ -290,7 +293,16 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
AddReplicaCmd.Response response = ocmh.addReplicaWithResp(clusterState, addReplicasProps, addResult);
- ocmh.overseer.getZkStateWriter().enqueueUpdate(response.clusterState, null,false);
+ DocCollection docColl = response.clusterState.getCollectionOrNull(coll.getName());
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(response.clusterState.getLiveNodes(), collectionStates);
+ ocmh.overseer.getZkStateWriter().enqueueUpdate(cs, null,false);
ocmh.overseer.writePendingUpdates();
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index b901efc..23bc23b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -258,7 +258,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
ClusterState clusterState = overseer.getZkStateWriter().getClusterstate(false);
@SuppressWarnings({"rawtypes"})
NamedList results = new NamedList();
-
+ String collection = message.getStr("collection");
+ if (collection == null) {
+ collection = message.getStr("name");
+ }
try {
CollectionAction action = getCollectionAction(operation);
Cmd command = commandMap.get(action);
@@ -271,7 +274,19 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
if (log.isDebugEnabled()) log.debug("Command returned clusterstate={} results={}", responce.clusterState, results);
if (responce.clusterState != null) {
- overseer.getZkStateWriter().enqueueUpdate(responce.clusterState, null, false);
+ DocCollection docColl = responce.clusterState.getCollectionOrNull(collection);
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ log.info("create new single collection state for collection {}", docColl.getName());
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ log.info("collection not found in returned state {} {}", collection, responce.clusterState);
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(responce.clusterState.getLiveNodes(), collectionStates);
+
+ overseer.getZkStateWriter().enqueueUpdate(cs, null, false);
overseer.writePendingUpdates();
}
@@ -281,15 +296,21 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
AddReplicaCmd.Response resp = responce.asyncFinalRunner.call();
if (log.isDebugEnabled()) log.debug("Finalize after Command returned clusterstate={}", resp.clusterState);
if (resp.clusterState != null) {
- overseer.getZkStateWriter().enqueueUpdate(responce.clusterState, null,false);
+ DocCollection docColl = resp.clusterState.getCollectionOrNull(collection);
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(responce.clusterState.getLiveNodes(), collectionStates);
+
+ overseer.getZkStateWriter().enqueueUpdate(cs, null,false);
overseer.writePendingUpdates();
}
}
- String collection = message.getStr("collection");
- if (collection == null) {
- collection = message.getStr("name");
- }
if (collection != null && responce.clusterState != null) {
Integer version = overseer.getZkStateWriter().lastWrittenVersion(collection);
if (version != null && !action.equals(DELETE)) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 0634699..4cb6dcf 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -79,6 +79,7 @@ import static org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Vari
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
@@ -347,8 +348,17 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
// });
// firstReplicaFutures.add(future);
}
+ DocCollection docColl = clusterState.getCollectionOrNull(message.getStr(COLLECTION));
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(clusterState.getLiveNodes(), collectionStates);
- ocmh.overseer.getZkStateWriter().enqueueUpdate(clusterState, null,false);
+ ocmh.overseer.getZkStateWriter().enqueueUpdate(cs, null,false);
ocmh.overseer.writePendingUpdates();
firstReplicaFutures.forEach(future -> {
try {
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index fcb77b0..2783361 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -90,13 +90,12 @@ public class ZkStateWriter {
this.reader = zkStateReader;
this.stats = stats;
- zkStateReader.forciblyRefreshAllClusterStateSlow();
cs = zkStateReader.getClusterState();
}
public void enqueueUpdate(ClusterState clusterState, ZkNodeProps message, boolean stateUpdate) throws Exception {
- if (log.isDebugEnabled()) log.debug("enqueue update stateUpdate={}", stateUpdate);
+ if (log.isDebugEnabled()) log.debug("enqueue update stateUpdate={} cs={}", stateUpdate, clusterState);
//log.info("Get our write lock for enq");
ourLock.lock();
//log.info("Got our write lock for enq");
@@ -123,12 +122,6 @@ public class ZkStateWriter {
Slice currentSlice = currentCollection.getSlice(slice.getName());
if (currentSlice != null) {
slice.setState(currentSlice.getState());
- Replica leader = currentSlice.getLeader();
- slice.setLeader(currentSlice.getLeader());
- if (leader != null) {
- leader.setState(Replica.State.ACTIVE);
- leader.getProperties().put("leader", "true");
- }
}
}