You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/09/17 10:22:07 UTC
[lucene-solr] branch jira/solr-14749 updated: SOLR-14749: More
javadoc, bug fixes & tests.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-14749
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/solr-14749 by this push:
new 9e4745c SOLR-14749: More javadoc, bug fixes & tests.
9e4745c is described below
commit 9e4745c41e4891f5f51e548c72e5d506c8444ee2
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Thu Sep 17 12:21:28 2020 +0200
SOLR-14749: More javadoc, bug fixes & tests.
---
.../apache/solr/cluster/events/ClusterEvent.java | 13 +++-
.../solr/cluster/events/ClusterEventProducer.java | 4 +-
...ent.java => ClusterPropertiesChangedEvent.java} | 10 +--
.../solr/cluster/events/CollectionsAddedEvent.java | 2 +-
.../cluster/events/CollectionsRemovedEvent.java | 2 +-
.../apache/solr/cluster/events/NodesDownEvent.java | 2 +-
.../apache/solr/cluster/events/NodesUpEvent.java | 2 +-
.../solr/cluster/events/ReplicasDownEvent.java | 2 +-
.../events/impl/ClusterEventProducerImpl.java | 18 ++---
.../impl/CollectionsRepairEventListener.java | 31 +++++++-
.../cluster/events/ClusterEventProducerTest.java | 84 ++++++++++++++++++++++
11 files changed, 146 insertions(+), 24 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java
index 2853f74..1929f0c 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/ClusterEvent.java
@@ -19,17 +19,23 @@ package org.apache.solr.cluster.events;
import java.time.Instant;
/**
- *
+ * Cluster-level event.
*/
public interface ClusterEvent {
enum EventType {
+ /** One or more nodes went down. */
NODES_DOWN,
+ /** One or more nodes went up. */
NODES_UP,
+ /** One or more collections have been added. */
COLLECTIONS_ADDED,
+ /** One or more collections have been removed. */
COLLECTIONS_REMOVED,
+ /** One or more replicas went down. */
REPLICAS_DOWN,
- CLUSTER_CONFIG_CHANGED,
+ /** Cluster properties have changed. */
+ CLUSTER_PROPERTIES_CHANGED
// other types? eg. Overseer leader change, shard leader change,
// node overload (eg. CPU / MEM circuit breakers tripped)?
}
@@ -37,6 +43,7 @@ public interface ClusterEvent {
/** Get event type. */
EventType getType();
- /** Get event timestamp. */
+ /** Get event timestamp. This is the instant when the event was generated (not necessarily when
+ * the underlying condition first occurred). */
Instant getTimestamp();
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ClusterEventProducer.java b/solr/core/src/java/org/apache/solr/cluster/events/ClusterEventProducer.java
index 0d01699..7832ee5 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ClusterEventProducer.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/ClusterEventProducer.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.cluster.events;
+import org.apache.solr.cloud.ClusterSingleton;
+
import java.util.Collections;
import java.util.Map;
import java.util.Set;
@@ -24,7 +26,7 @@ import java.util.concurrent.ConcurrentHashMap;
/**
* Component that produces {@link ClusterEvent} instances.
*/
-public interface ClusterEventProducer {
+public interface ClusterEventProducer extends ClusterSingleton {
String PLUGIN_NAME = "clusterEventProducer";
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ClusterConfigChangedEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/ClusterPropertiesChangedEvent.java
similarity index 74%
rename from solr/core/src/java/org/apache/solr/cluster/events/ClusterConfigChangedEvent.java
rename to solr/core/src/java/org/apache/solr/cluster/events/ClusterPropertiesChangedEvent.java
index e536468..0906e7e 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ClusterConfigChangedEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/ClusterPropertiesChangedEvent.java
@@ -19,17 +19,17 @@ package org.apache.solr.cluster.events;
import java.util.Map;
/**
- *
+ * Event generated when {@link org.apache.solr.common.cloud.ZkStateReader#CLUSTER_PROPS} is modified.
*/
-public interface ClusterConfigChangedEvent extends ClusterEvent {
+public interface ClusterPropertiesChangedEvent extends ClusterEvent {
@Override
default EventType getType() {
- return EventType.CLUSTER_CONFIG_CHANGED;
+ return EventType.CLUSTER_PROPERTIES_CHANGED;
}
- Map<String, Object> getOldClusterConfig();
+ Map<String, Object> getOldClusterProperties();
- Map<String, Object> getNewClusterConfig();
+ Map<String, Object> getNewClusterProperties();
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/CollectionsAddedEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/CollectionsAddedEvent.java
index f4ba87e..baeae52 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/CollectionsAddedEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/CollectionsAddedEvent.java
@@ -19,7 +19,7 @@ package org.apache.solr.cluster.events;
import java.util.Collection;
/**
- *
+ * Event generated when some collections have been added.
*/
public interface CollectionsAddedEvent extends ClusterEvent {
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/CollectionsRemovedEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/CollectionsRemovedEvent.java
index e6a9175..36327a9 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/CollectionsRemovedEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/CollectionsRemovedEvent.java
@@ -19,7 +19,7 @@ package org.apache.solr.cluster.events;
import java.util.Collection;
/**
- *
+ * Event generated when some collections have been removed.
*/
public interface CollectionsRemovedEvent extends ClusterEvent {
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/NodesDownEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/NodesDownEvent.java
index 06141e3..d749ac8 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/NodesDownEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/NodesDownEvent.java
@@ -19,7 +19,7 @@ package org.apache.solr.cluster.events;
import java.util.Collection;
/**
- *
+ * Event generated when some nodes went down.
*/
public interface NodesDownEvent extends ClusterEvent {
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/NodesUpEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/NodesUpEvent.java
index 241b8bf..c947aac 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/NodesUpEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/NodesUpEvent.java
@@ -19,7 +19,7 @@ package org.apache.solr.cluster.events;
import java.util.Collection;
/**
- *
+ * Event generated when some nodes went up.
*/
public interface NodesUpEvent extends ClusterEvent {
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/ReplicasDownEvent.java b/solr/core/src/java/org/apache/solr/cluster/events/ReplicasDownEvent.java
index e4dc106..5d2c3e7 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/ReplicasDownEvent.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/ReplicasDownEvent.java
@@ -21,7 +21,7 @@ import org.apache.solr.common.cloud.Replica;
import java.util.Collection;
/**
- *
+ * Event generated when some replicas went down.
*/
public interface ReplicasDownEvent extends ClusterEvent {
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java b/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java
index a6897e7..d8f5082 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerImpl.java
@@ -24,11 +24,12 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.solr.cloud.ZkController;
-import org.apache.solr.cluster.events.ClusterConfigChangedEvent;
+import org.apache.solr.cluster.events.ClusterPropertiesChangedEvent;
import org.apache.solr.cluster.events.ClusterEvent;
import org.apache.solr.cluster.events.ClusterEventListener;
import org.apache.solr.cluster.events.ClusterEventProducer;
@@ -69,7 +70,7 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
ClusterEvent.EventType.NODES_UP,
ClusterEvent.EventType.COLLECTIONS_ADDED,
ClusterEvent.EventType.COLLECTIONS_REMOVED,
- ClusterEvent.EventType.CLUSTER_CONFIG_CHANGED
+ ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED
));
private volatile boolean isClosed = false;
@@ -176,19 +177,20 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
});
zkController.zkStateReader.registerCloudCollectionsListener(cloudCollectionsListener);
- lastClusterProperties = zkController.zkStateReader.getClusterProperties();
+ lastClusterProperties = new LinkedHashMap<>(zkController.zkStateReader.getClusterProperties());
clusterPropertiesListener = (newProperties) -> {
if (newProperties.equals(lastClusterProperties)) {
return false;
}
- fireEvent(new ClusterConfigChangedEvent() {
+ fireEvent(new ClusterPropertiesChangedEvent() {
+ final Map<String, Object> oldProps = lastClusterProperties;
@Override
- public Map<String, Object> getOldClusterConfig() {
- return lastClusterProperties;
+ public Map<String, Object> getOldClusterProperties() {
+ return oldProps;
}
@Override
- public Map<String, Object> getNewClusterConfig() {
+ public Map<String, Object> getNewClusterProperties() {
return newProperties;
}
@@ -197,7 +199,7 @@ public class ClusterEventProducerImpl implements ClusterEventProducer, ClusterSi
return Instant.now();
}
});
- lastClusterProperties = newProperties;
+ lastClusterProperties = new LinkedHashMap<>(newProperties);
return false;
};
zkController.zkStateReader.registerClusterPropertiesListener(clusterPropertiesListener);
diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java b/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java
index 773362a..0c6ce9c 100644
--- a/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java
+++ b/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java
@@ -16,17 +16,24 @@
*/
package org.apache.solr.cluster.events.impl;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
+import org.apache.solr.client.solrj.cloud.NodeStateProvider;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.cluster.events.ClusterEvent;
import org.apache.solr.cluster.events.ClusterEventListener;
import org.apache.solr.cloud.ClusterSingleton;
import org.apache.solr.cluster.events.NodesDownEvent;
import org.apache.solr.cluster.events.ReplicasDownEvent;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.core.CoreContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,12 +53,14 @@ public class CollectionsRepairEventListener implements ClusterSingleton, Cluster
private final CoreContainer cc;
private final SolrClientCache solrClientCache;
+ private final SolrCloudManager solrCloudManager;
private boolean running = false;
public CollectionsRepairEventListener(CoreContainer cc) {
this.cc = cc;
this.solrClientCache = cc.getSolrClientCache();
+ this.solrCloudManager = cc.getZkController().getSolrCloudManager();
}
@Override
@@ -81,11 +90,29 @@ public class CollectionsRepairEventListener implements ClusterSingleton, Cluster
}
private void handleNodesDown(NodesDownEvent event) {
- // send MOVEREPLICA admin requests for all replicas from that node
+ // collect all lost replicas
+ List<Replica> lostReplicas = new ArrayList<>();
+ try {
+ ClusterState clusterState = solrCloudManager.getClusterStateProvider().getClusterState();
+ clusterState.forEachCollection(coll -> {
+ coll.forEachReplica((shard, replica) -> {
+ if (event.getNodeNames().contains(replica.getNodeName())) {
+ lostReplicas.add(replica);
+ }
+ });
+ });
+ } catch (IOException e) {
+ log.warn("Exception getting cluster state", e);
+ return;
+ }
+
+ // compute new placements for all replicas from lost nodes
+ // send MOVEREPLICA admin requests for each lost replica
}
private void handleReplicasDown(ReplicasDownEvent event) {
- // send ADDREPLICA admin request
+ // compute new placements for all replicas that went down
+ // send ADDREPLICA admin request for each lost replica
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java b/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java
index bde8ca4..0e39e8f 100644
--- a/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/events/ClusterEventProducerTest.java
@@ -1,7 +1,9 @@
package org.apache.solr.cluster.events;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.ClusterProperties;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -13,6 +15,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
/**
*
@@ -20,6 +24,8 @@ import java.util.Set;
public class ClusterEventProducerTest extends SolrCloudTestCase {
public static class AllEventsListener implements ClusterEventListener {
+ CountDownLatch eventLatch = new CountDownLatch(1);
+ ClusterEvent.EventType expectedType;
Map<ClusterEvent.EventType, List<ClusterEvent>> events = new HashMap<>();
@Override
@@ -30,6 +36,21 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
@Override
public void onEvent(ClusterEvent event) {
events.computeIfAbsent(event.getType(), type -> new ArrayList<>()).add(event);
+ if (event.getType() == expectedType) {
+ eventLatch.countDown();
+ }
+ }
+
+ public void setExpectedType(ClusterEvent.EventType expectedType) {
+ this.expectedType = expectedType;
+ eventLatch = new CountDownLatch(1);
+ }
+
+ public void waitForExpectedEvent(int timeoutSeconds) throws InterruptedException {
+ boolean await = eventLatch.await(timeoutSeconds, TimeUnit.SECONDS);
+ if (!await) {
+ fail("Timed out waiting for expected event " + expectedType);
+ }
}
}
@@ -54,6 +75,8 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
// NODES_DOWN
+ eventsListener.setExpectedType(ClusterEvent.EventType.NODES_DOWN);
+
// don't kill Overseer
JettySolrRunner nonOverseerJetty = null;
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
@@ -66,6 +89,7 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
String nodeName = nonOverseerJetty.getNodeName();
cluster.stopJettySolrRunner(nonOverseerJetty);
cluster.waitForJettyToStop(nonOverseerJetty);
+ eventsListener.waitForExpectedEvent(10);
assertNotNull("should be NODES_DOWN events", eventsListener.events.get(ClusterEvent.EventType.NODES_DOWN));
List<ClusterEvent> events = eventsListener.events.get(ClusterEvent.EventType.NODES_DOWN);
assertEquals("should be one NODES_DOWN event", 1, events.size());
@@ -75,8 +99,10 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
assertEquals("should be node " + nodeName, nodeName, nodesDown.getNodeNames().iterator().next());
// NODES_UP
+ eventsListener.setExpectedType(ClusterEvent.EventType.NODES_UP);
JettySolrRunner newNode = cluster.startJettySolrRunner();
cluster.waitForNode(newNode, 60);
+ eventsListener.waitForExpectedEvent(10);
assertNotNull("should be NODES_UP events", eventsListener.events.get(ClusterEvent.EventType.NODES_UP));
events = eventsListener.events.get(ClusterEvent.EventType.NODES_UP);
assertEquals("should be one NODES_UP event", 1, events.size());
@@ -85,6 +111,64 @@ public class ClusterEventProducerTest extends SolrCloudTestCase {
NodesUpEvent nodesUp = (NodesUpEvent) event;
assertEquals("should be node " + newNode.getNodeName(), newNode.getNodeName(), nodesUp.getNodeNames().iterator().next());
+ // COLLECTIONS_ADDED
+ eventsListener.setExpectedType(ClusterEvent.EventType.COLLECTIONS_ADDED);
+ String collection = "testNodesEvent_collection";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collection, "conf", 1, 1);
+ cluster.getSolrClient().request(create);
+ cluster.waitForActiveCollection(collection, 1, 1);
+ eventsListener.waitForExpectedEvent(10);
+ assertNotNull("should be COLLECTIONS_ADDED events", eventsListener.events.get(ClusterEvent.EventType.COLLECTIONS_ADDED));
+ events = eventsListener.events.get(ClusterEvent.EventType.COLLECTIONS_ADDED);
+ assertEquals("should be one COLLECTIONS_ADDED event", 1, events.size());
+ event = events.get(0);
+ assertEquals("should be COLLECTIONS_ADDED event type", ClusterEvent.EventType.COLLECTIONS_ADDED, event.getType());
+ CollectionsAddedEvent collectionsAdded = (CollectionsAddedEvent) event;
+ assertEquals("should be collection " + collection, collection, collectionsAdded.getCollectionNames().iterator().next());
+
+ // COLLECTIONS_REMOVED
+ eventsListener.setExpectedType(ClusterEvent.EventType.COLLECTIONS_REMOVED);
+ CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collection);
+ cluster.getSolrClient().request(delete);
+ eventsListener.waitForExpectedEvent(10);
+ assertNotNull("should be COLLECTIONS_REMOVED events", eventsListener.events.get(ClusterEvent.EventType.COLLECTIONS_REMOVED));
+ events = eventsListener.events.get(ClusterEvent.EventType.COLLECTIONS_REMOVED);
+ assertEquals("should be one COLLECTIONS_REMOVED event", 1, events.size());
+ event = events.get(0);
+ assertEquals("should be COLLECTIONS_REMOVED event type", ClusterEvent.EventType.COLLECTIONS_REMOVED, event.getType());
+ CollectionsRemovedEvent collectionsRemoved = (CollectionsRemovedEvent) event;
+ assertEquals("should be collection " + collection, collection, collectionsRemoved.getCollectionNames().iterator().next());
+
+ // CLUSTER_CONFIG_CHANGED
+ eventsListener.setExpectedType(ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED);
+ ClusterProperties clusterProperties = new ClusterProperties(cluster.getZkClient());
+ Map<String, Object> oldProps = new HashMap<>(clusterProperties.getClusterProperties());
+ clusterProperties.setClusterProperty("ext.foo", "bar");
+ eventsListener.waitForExpectedEvent(10);
+ assertNotNull("should be CLUSTER_CONFIG_CHANGED events", eventsListener.events.get(ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED));
+ events = eventsListener.events.get(ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED);
+ assertEquals("should be one CLUSTER_CONFIG_CHANGED event", 1, events.size());
+ event = events.get(0);
+ assertEquals("should be CLUSTER_CONFIG_CHANGED event type", ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED, event.getType());
+ ClusterPropertiesChangedEvent propertiesChanged = (ClusterPropertiesChangedEvent) event;
+ Map<String, Object> newProps = propertiesChanged.getNewClusterProperties();
+ assertEquals("old props should be the same", oldProps, propertiesChanged.getOldClusterProperties());
+ assertEquals("new properties wrong value of the 'ext.foo' property: " + newProps,
+ "bar", newProps.get("ext.foo"));
+
+ // unset the property
+ eventsListener.setExpectedType(ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED);
+ clusterProperties.setClusterProperty("ext.foo", null);
+ eventsListener.waitForExpectedEvent(10);
+ assertNotNull("should be CLUSTER_CONFIG_CHANGED events", eventsListener.events.get(ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED));
+ events = eventsListener.events.get(ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED);
+ assertEquals("should be two CLUSTER_CONFIG_CHANGED events", 2, events.size());
+ event = events.get(1);
+ assertEquals("should be CLUSTER_CONFIG_CHANGED event type", ClusterEvent.EventType.CLUSTER_PROPERTIES_CHANGED, event.getType());
+ propertiesChanged = (ClusterPropertiesChangedEvent) event;
+ assertEquals("old props should be the same as previous new props", newProps, propertiesChanged.getOldClusterProperties());
+ assertEquals("new properties should not have 'ext.foo' property: " + propertiesChanged.getNewClusterProperties(),
+ null, propertiesChanged.getNewClusterProperties().get("ext.foo"));
}
}