You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/11/11 23:21:14 UTC
[lucene-solr] branch reference_impl_dev updated (3b02c91 -> dbc91c2)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 3b02c91 @1167 Some state publish improvements.
new 8cce4cf @1168 Logging cleanup.
new 99fbe9d @1169 Pull unnecessary properties from state.json.
new dbc91c2 @1170 Only dirty the collections we change.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/java/org/apache/solr/cloud/Overseer.java | 57 ----------------------
.../solr/cloud/OverseerTaskExecutorTask.java | 3 --
.../cloud/api/collections/CreateCollectionCmd.java | 3 +-
.../cloud/api/collections/DeleteReplicaCmd.java | 6 +--
.../solr/cloud/api/collections/MoveReplicaCmd.java | 14 +++++-
.../OverseerCollectionMessageHandler.java | 35 ++++++++++---
.../solr/cloud/api/collections/SplitShardCmd.java | 12 ++++-
.../apache/solr/cloud/overseer/ZkStateWriter.java | 18 +++----
.../apache/solr/cloud/CollectionsAPISolrJTest.java | 11 +----
.../solrj/request/CollectionAdminRequest.java | 4 +-
10 files changed, 69 insertions(+), 94 deletions(-)
[lucene-solr] 01/03: @1168 Logging cleanup.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 8cce4cf7508a9ced8f5569c3af5cbba379cf7712
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Nov 11 16:09:55 2020 -0600
@1168 Logging cleanup.
---
.../src/java/org/apache/solr/cloud/Overseer.java | 57 ----------------------
.../cloud/api/collections/DeleteReplicaCmd.java | 6 +--
2 files changed, 3 insertions(+), 60 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 1b31274..39a7d1b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -732,63 +732,6 @@ public class Overseer implements SolrCloseable {
}
- // nocommit use
- private static class OverseerWatcher implements Watcher, Closeable {
-
- private final CoreContainer cc;
- private final ZkController zkController;
- private final String path = Overseer.OVERSEER_ELECT + "/leader";
- private final Overseer overseer;
-
- public OverseerWatcher(CoreContainer cc) {
- this.cc = cc;
- this.zkController = cc.getZkController();
- this.overseer = zkController.getOverseer();
- }
-
- @Override
- public void process(WatchedEvent event) {
- try {
- if (log.isDebugEnabled()) {
- log.debug("set watch on leader znode");
- }
- zkController.getZkClient().exists(path, new Watcher() {
-
- @Override
- public void process(WatchedEvent event) {
- if (Event.EventType.None.equals(event.getType())) {
- return;
- }
-
- log.info("Overseer leader has changed, closing ...");
- overseer.close();
-
- }}, true);
- } catch (KeeperException.SessionExpiredException e) {
- log.warn("ZooKeeper session expired");
- overseer.doClose();
- return;
- } catch (InterruptedException | AlreadyClosedException e) {
- log.info("Already closed");
- overseer.doClose();
- return;
- } catch (Exception e) {
- log.error("Unexpected error in Overseer state update loop", e);
- overseer.doClose();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- zkController.getZkClient().getSolrZooKeeper().removeWatches(path, this, WatcherType.Data, true);
- } catch (Exception e) {
- log.info("", e.getMessage());
- }
- }
- }
-
private static abstract class QueueWatcher implements Watcher, Closeable {
protected final CoreContainer cc;
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
index 91634d7..1be60ef 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
@@ -294,7 +294,7 @@ public class DeleteReplicaCmd implements Cmd {
replicasToBeRemoved.add(replica.getName());
count --;
}
- log.info("Found replicas to be removed {}", replicasToBeRemoved);
+ if (log.isDebugEnabled()) log.debug("Found replicas to be removed {}", replicasToBeRemoved);
return replicasToBeRemoved;
}
@@ -350,9 +350,9 @@ public class DeleteReplicaCmd implements Cmd {
rep.getProperties().put("collection", replica.getCollection());
rep.getProperties().put(ZkStateReader.NODE_NAME_PROP, replica.getNodeName());
- log.info("Before slice remove replica {} {}", rep, clusterState);
+ if (log.isDebugEnabled()) log.debug("Before slice remove replica {} {}", rep, clusterState);
clusterState = new SliceMutator(ocmh.cloudManager).removeReplica(clusterState, rep);
- log.info("After slice remove replica {} {}", rep, clusterState);
+ if (log.isDebugEnabled()) log.debug("After slice remove replica {} {}", rep, clusterState);
boolean isLive = false;
if (!onlyUpdateState) {
[lucene-solr] 02/03: @1169 Pull unnecessary properties from
state.json.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 99fbe9d33e6091099e4e7ce0ff2b6ac2dd5ad200
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Nov 11 16:24:42 2020 -0600
@1169 Pull unnecessary properties from state.json.
---
.../java/org/apache/solr/cloud/overseer/ZkStateWriter.java | 9 ++++++++-
.../test/org/apache/solr/cloud/CollectionsAPISolrJTest.java | 11 ++---------
.../solr/client/solrj/request/CollectionAdminRequest.java | 4 +---
3 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 13bb6cb..fcb77b0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -107,10 +107,17 @@ public class ZkStateWriter {
if (clusterState == null) {
throw new NullPointerException("clusterState cannot be null");
}
+
+
+
changed.set(true);
clusterState.forEachCollection(collection -> {
DocCollection currentCollection = cs.getCollectionOrNull(collection.getName());
-
+ collection.getProperties().remove("pullReplicas");
+ collection.getProperties().remove("replicationFactor");
+ collection.getProperties().remove("maxShardsPerNode");
+ collection.getProperties().remove("nrtReplicas");
+ collection.getProperties().remove("tlogReplicas");
for (Slice slice : collection) {
if (currentCollection != null) {
Slice currentSlice = currentCollection.getSlice(slice.getName());
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 54bda46..230f34a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -832,18 +832,11 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
.process(cluster.getSolrClient());
CollectionAdminRequest.modifyCollection(collection, null)
- .setAttribute("replicationFactor", 25)
- .process(cluster.getSolrClient());
-
- waitForState("Expecting attribute 'replicationFactor' to be 25", collection,
- (n, c) -> 25 == c.getReplicationFactor());
-
- CollectionAdminRequest.modifyCollection(collection, null)
- .unsetAttribute("maxShardsPerNode")
+ .unsetAttribute("router")
.process(cluster.getSolrClient());
waitForState("Expecting attribute 'maxShardsPerNode' to be deleted", collection,
- (n, c) -> null == c.get("maxShardsPerNode"));
+ (n, c) -> null == c.get("router"));
expectThrows(IllegalArgumentException.class,
"An attempt to set unknown collection attribute should have failed",
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 6754d70..2ecde29 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -83,12 +83,10 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
* The set of modifiable collection properties
*/
public static final java.util.List<String> MODIFIABLE_COLLECTION_PROPERTIES = Arrays.asList(
- REPLICATION_FACTOR,
- MAX_SHARDS_PER_NODE,
COLL_CONF,
WITH_COLLECTION,
COLOCATED_WITH,
- READ_ONLY);
+ READ_ONLY, "router");
protected final CollectionAction action;
[lucene-solr] 03/03: @1170 Only dirty the collections we change.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit dbc91c2ee62506fb6a4c7c1f7010a5c21afa0413
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Nov 11 17:07:43 2020 -0600
@1170 Only dirty the collections we change.
---
.../solr/cloud/OverseerTaskExecutorTask.java | 3 --
.../cloud/api/collections/CreateCollectionCmd.java | 3 +-
.../solr/cloud/api/collections/MoveReplicaCmd.java | 14 ++++++++-
.../OverseerCollectionMessageHandler.java | 35 +++++++++++++++++-----
.../solr/cloud/api/collections/SplitShardCmd.java | 12 +++++++-
.../apache/solr/cloud/overseer/ZkStateWriter.java | 9 +-----
6 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java
index 2277034..52840ca 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskExecutorTask.java
@@ -70,10 +70,7 @@ public class OverseerTaskExecutorTask implements Runnable {
log.info("Queue operation is {}", operation);
- ClusterState cs = zkStateWriter.getClusterstate(true);
-
log.info("Process message {} {}", message, operation);
- // ClusterState newClusterState = processMessage(message, operation, cs);
log.info("Enqueue message {}", operation);
zkStateWriter.enqueueUpdate(null, message, true);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 9e29e30..4aaaee8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -331,7 +331,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
log.error("Exception trying to clean up collection after fail {}", collectionName);
}
if (log.isDebugEnabled()) log.debug("Cleaned up artifacts for failed create collection for [{}]", collectionName);
- //throw new SolrException(ErrorCode.BAD_REQUEST, "Underlying core creation failed while creating collection: " + collectionName + "\n" + results);
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Underlying core creation failed while creating collection: " + collectionName + "\n" + results);
} else {
Object createNodeSet = message.get(ZkStateReader.CREATE_NODE_SET);
if (log.isDebugEnabled()) log.debug("createNodeSet={}", createNodeSet);
@@ -379,6 +379,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
};
}
+ if (log.isDebugEnabled()) log.debug("return cs from create collection cmd {}", clusterState);
response.clusterState = clusterState;
return response;
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
index 691e267..a9c5a0b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
@@ -20,8 +20,10 @@ package org.apache.solr.cloud.api.collections;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -49,6 +51,7 @@ import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHan
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
@@ -290,7 +293,16 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
AddReplicaCmd.Response response = ocmh.addReplicaWithResp(clusterState, addReplicasProps, addResult);
- ocmh.overseer.getZkStateWriter().enqueueUpdate(response.clusterState, null,false);
+ DocCollection docColl = response.clusterState.getCollectionOrNull(coll.getName());
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(response.clusterState.getLiveNodes(), collectionStates);
+ ocmh.overseer.getZkStateWriter().enqueueUpdate(cs, null,false);
ocmh.overseer.writePendingUpdates();
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index b901efc..23bc23b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -258,7 +258,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
ClusterState clusterState = overseer.getZkStateWriter().getClusterstate(false);
@SuppressWarnings({"rawtypes"})
NamedList results = new NamedList();
-
+ String collection = message.getStr("collection");
+ if (collection == null) {
+ collection = message.getStr("name");
+ }
try {
CollectionAction action = getCollectionAction(operation);
Cmd command = commandMap.get(action);
@@ -271,7 +274,19 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
if (log.isDebugEnabled()) log.debug("Command returned clusterstate={} results={}", responce.clusterState, results);
if (responce.clusterState != null) {
- overseer.getZkStateWriter().enqueueUpdate(responce.clusterState, null, false);
+ DocCollection docColl = responce.clusterState.getCollectionOrNull(collection);
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ log.info("create new single collection state for collection {}", docColl.getName());
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ log.info("collection not found in returned state {} {}", collection, responce.clusterState);
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(responce.clusterState.getLiveNodes(), collectionStates);
+
+ overseer.getZkStateWriter().enqueueUpdate(cs, null, false);
overseer.writePendingUpdates();
}
@@ -281,15 +296,21 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
AddReplicaCmd.Response resp = responce.asyncFinalRunner.call();
if (log.isDebugEnabled()) log.debug("Finalize after Command returned clusterstate={}", resp.clusterState);
if (resp.clusterState != null) {
- overseer.getZkStateWriter().enqueueUpdate(responce.clusterState, null,false);
+ DocCollection docColl = resp.clusterState.getCollectionOrNull(collection);
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(responce.clusterState.getLiveNodes(), collectionStates);
+
+ overseer.getZkStateWriter().enqueueUpdate(cs, null,false);
overseer.writePendingUpdates();
}
}
- String collection = message.getStr("collection");
- if (collection == null) {
- collection = message.getStr("name");
- }
if (collection != null && responce.clusterState != null) {
Integer version = overseer.getZkStateWriter().lastWrittenVersion(collection);
if (version != null && !action.equals(DELETE)) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 0634699..4cb6dcf 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -79,6 +79,7 @@ import static org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Vari
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
@@ -347,8 +348,17 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
// });
// firstReplicaFutures.add(future);
}
+ DocCollection docColl = clusterState.getCollectionOrNull(message.getStr(COLLECTION));
+ Map<String, DocCollection> collectionStates;
+ if (docColl != null) {
+ collectionStates = new HashMap<>();
+ collectionStates.put(docColl.getName(), docColl);
+ } else {
+ collectionStates = new HashMap<>();
+ }
+ ClusterState cs = new ClusterState(clusterState.getLiveNodes(), collectionStates);
- ocmh.overseer.getZkStateWriter().enqueueUpdate(clusterState, null,false);
+ ocmh.overseer.getZkStateWriter().enqueueUpdate(cs, null,false);
ocmh.overseer.writePendingUpdates();
firstReplicaFutures.forEach(future -> {
try {
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index fcb77b0..2783361 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -90,13 +90,12 @@ public class ZkStateWriter {
this.reader = zkStateReader;
this.stats = stats;
- zkStateReader.forciblyRefreshAllClusterStateSlow();
cs = zkStateReader.getClusterState();
}
public void enqueueUpdate(ClusterState clusterState, ZkNodeProps message, boolean stateUpdate) throws Exception {
- if (log.isDebugEnabled()) log.debug("enqueue update stateUpdate={}", stateUpdate);
+ if (log.isDebugEnabled()) log.debug("enqueue update stateUpdate={} cs={}", stateUpdate, clusterState);
//log.info("Get our write lock for enq");
ourLock.lock();
//log.info("Got our write lock for enq");
@@ -123,12 +122,6 @@ public class ZkStateWriter {
Slice currentSlice = currentCollection.getSlice(slice.getName());
if (currentSlice != null) {
slice.setState(currentSlice.getState());
- Replica leader = currentSlice.getLeader();
- slice.setLeader(currentSlice.getLeader());
- if (leader != null) {
- leader.setState(Replica.State.ACTIVE);
- leader.getProperties().put("leader", "true");
- }
}
}