You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/12/01 15:28:37 UTC
lucene-solr:jira/solr-11285-sim: SOLR-11285: Add some javadocs. Try
to test searchRate trigger (still broken).
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11285-sim d27dd2105 -> 5d2524e9b
SOLR-11285: Add some javadocs. Try to test searchRate trigger (still broken).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5d2524e9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5d2524e9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5d2524e9
Branch: refs/heads/jira/solr-11285-sim
Commit: 5d2524e9b198ee062c8f9cde711889af190b885c
Parents: d27dd21
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Fri Dec 1 16:28:07 2017 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Fri Dec 1 16:28:07 2017 +0100
----------------------------------------------------------------------
.../cloud/autoscaling/ComputePlanAction.java | 12 +-
.../cloud/autoscaling/SearchRateTrigger.java | 5 +-
.../autoscaling/TriggerIntegrationTest.java | 7 +
.../solr/cloud/autoscaling/sim/ActionError.java | 2 +-
.../sim/GenericDistributedQueue.java | 2 +-
.../cloud/autoscaling/sim/SimCloudManager.java | 46 ++++-
.../sim/SimClusterStateProvider.java | 207 +++++++++++++++++--
.../sim/SimDistributedQueueFactory.java | 4 +-
.../autoscaling/sim/SimNodeStateProvider.java | 45 +++-
.../cloud/autoscaling/sim/TestLargeCluster.java | 62 +++++-
10 files changed, 361 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 9eed1f2..c770cac 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -35,6 +35,7 @@ import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,13 +113,10 @@ public class ComputePlanAction extends TriggerActionBase {
} else {
// collection || shard || replica -> ADDREPLICA
suggester = session.getSuggester(CollectionParams.CollectionAction.ADDREPLICA);
- Set<String> collections = new HashSet<>();
- // XXX improve this when AddReplicaSuggester supports coll_shard hint
- hotReplicas.forEach(r -> collections.add(r.getCollection()));
- hotShards.forEach((coll, shards) -> collections.add(coll));
- hotCollections.forEach((coll, rate) -> collections.add(coll));
- for (String coll : collections) {
- suggester = suggester.hint(Suggester.Hint.COLL, coll);
+ Set<Pair> collectionShards = new HashSet<>();
+ hotShards.forEach((coll, shards) -> shards.forEach((s, r) -> collectionShards.add(new Pair(coll, s))));
+ for (Pair<String, String> colShard : collectionShards) {
+ suggester = suggester.hint(Suggester.Hint.COLL_SHARD, colShard);
}
}
break;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index 1e0caec..ec3110e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@ -165,6 +165,9 @@ public class SearchRateTrigger extends TriggerBase {
});
});
});
+ if (metricTags.isEmpty()) {
+ continue;
+ }
Map<String, Object> rates = cloudManager.getNodeStateProvider().getNodeValues(node, metricTags.keySet());
rates.forEach((tag, rate) -> {
ReplicaInfo info = metricTags.get(tag);
@@ -271,7 +274,7 @@ public class SearchRateTrigger extends TriggerBase {
private boolean waitForElapsed(String name, long now, Map<String, Long> lastEventMap) {
Long lastTime = lastEventMap.computeIfAbsent(name, s -> now);
long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS);
- log.debug("name=" + name + ", lastTime=" + lastTime + ", elapsed=" + elapsed);
+ log.trace("name={}, lastTime={}, elapsed={}", name, lastTime, elapsed);
if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index f7e1a62..ebd7cbd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -1394,6 +1394,10 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Test
public void testSearchRate() throws Exception {
+ // start a few more jetty-s
+ for (int i = 0; i < 3; i++) {
+ cluster.startJettySolrRunner();
+ }
CloudSolrClient solrClient = cluster.getSolrClient();
String COLL1 = "collection1";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
@@ -1407,6 +1411,8 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
"'enabled' : true," +
"'rate' : 1.0," +
"'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
"{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" +
"]" +
"}}";
@@ -1420,6 +1426,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
"'name' : 'srt'," +
"'trigger' : 'search_rate_trigger'," +
"'stage' : ['FAILED','SUCCEEDED']," +
+ "'afterAction': ['compute', 'execute', 'test']," +
"'class' : '" + TestTriggerListener.class.getName() + "'" +
"}" +
"}";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
index c10b42e..c1c070d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
@@ -17,7 +17,7 @@
package org.apache.solr.cloud.autoscaling.sim;
/**
- *
+ * Interface that helps simulating action errors.
*/
public interface ActionError {
boolean shouldFail(String... args);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
index ee7e839..5d7aa4d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
@@ -53,7 +53,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A distributed queue that uses {@link org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager}.
+ * A distributed queue that uses {@link DistribStateManager} as the underlying distributed store.
* Implementation based on {@link org.apache.solr.cloud.ZkDistributedQueue}
*/
public class GenericDistributedQueue implements DistributedQueue {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index 4a37e35..ecfc8f7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -159,6 +159,11 @@ public class SimCloudManager implements SolrCloudManager {
return cloudManager;
}
+ /**
+ * Create node values (metrics) for a node.
+ * @param nodeName node name (eg. '127.0.0.1:10000_solr')
+ * @return node values
+ */
public static Map<String, Object> createNodeValues(String nodeName) {
Map<String, Object> values = new HashMap<>();
String host, nodeId;
@@ -206,6 +211,10 @@ public class SimCloudManager implements SolrCloudManager {
return values;
}
+ /**
+ * Add a new node and initialize its node values (metrics).
+ * @return new node id
+ */
public String simAddNode() throws Exception {
Map<String, Object> values = createNodeValues(null);
String nodeId = (String)values.get(ImplicitSnitch.NODE);
@@ -215,12 +224,21 @@ public class SimCloudManager implements SolrCloudManager {
return nodeId;
}
+ /**
+ * Remove a node from the cluster. This simulates a node lost scenario.
+ * @param nodeId node id
+ */
public void simRemoveNode(String nodeId) throws Exception {
clusterStateProvider.simRemoveNode(nodeId);
nodeStateProvider.simRemoveNodeValues(nodeId);
LOG.trace("-- removed node " + nodeId);
}
+ /**
+ * Remove a number of randomly selected nodes
+ * @param number number of nodes to remove
+ * @param random random
+ */
public void simRemoveRandomNodes(int number, Random random) throws Exception {
List<String> nodes = new ArrayList<>(liveNodes);
Collections.shuffle(nodes, random);
@@ -230,14 +248,26 @@ public class SimCloudManager implements SolrCloudManager {
}
}
+ /**
+ * Clear the (simulated) .system collection.
+ */
public void simClearSystemCollection() {
systemColl.clear();
}
+ /**
+ * Get the content of (simulated) .system collection.
+ * @return documents in the collection.
+ */
public List<SolrInputDocument> simGetSystemCollection() {
return systemColl;
}
+ /**
+ * Get a {@link SolrClient} implementation where calls are forwarded to this
+ * instance.
+ * @return simulated SolrClient.
+ */
public SolrClient simGetSolrClient() {
if (solrClient != null) {
return solrClient;
@@ -257,6 +287,11 @@ public class SimCloudManager implements SolrCloudManager {
}
}
+ /**
+ * Submit a task to execute in a thread pool.
+ * @param callable task to execute
+ * @return future to obtain results
+ */
public <T> Future<T> submit(Callable<T> callable) {
return simCloudManagerPool.submit(callable);
}
@@ -325,6 +360,12 @@ public class SimCloudManager implements SolrCloudManager {
}
}
+ /**
+ * Handler for autoscaling requests. NOTE: only a specific subset of autoscaling requests is
+ * supported!
+ * @param req autoscaling request
+ * @return results
+ */
public SolrResponse simHandleSolrRequest(SolrRequest req) throws IOException, InterruptedException {
// pay the penalty for remote request, at least 5 ms
timeSource.sleep(5);
@@ -466,8 +507,9 @@ public class SimCloudManager implements SolrCloudManager {
}
-
-
+ /**
+ * HTTP requests are not supported by this implementation.
+ */
@Override
public byte[] httpRequest(String url, SolrRequest.METHOD method, Map<String, String> headers, String payload, int timeout, boolean followRedirects) throws IOException {
throw new UnsupportedOperationException("general HTTP requests are not supported yet");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 1f1788b..f65f31a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -105,8 +105,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
private AtomicReference<Map<String, DocCollection>> collectionsStatesRef = new AtomicReference<>();
/**
- * Zero-arg constructor. The instance needs to be initialized using the <code>sim*</code> methods in order
- * to ensure proper behavior, otherwise it will behave as a cluster with zero live nodes and zero replicas.
+ * The instance needs to be initialized using the <code>sim*</code> methods in order
+ * to ensure proper behavior, otherwise it will behave as a cluster with zero replicas.
*/
public SimClusterStateProvider(Set<String> liveNodes, SimCloudManager cloudManager) {
this.liveNodes = liveNodes;
@@ -126,6 +126,10 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// ============== SIMULATOR SETUP METHODS ====================
+ /**
+ * Initialize from an existing cluster state
+ * @param initialState initial cluster state
+ */
public void simSetClusterState(ClusterState initialState) throws Exception {
lock.lock();
try {
@@ -154,10 +158,18 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Reset the leader election throttle.
+ */
public void simResetLeaderThrottle() {
leaderThrottle.reset();
}
+ /**
+ * Get random node id.
+ * @param random instance of random.
+ * @return one of the live nodes
+ */
public String simGetRandomNode(Random random) {
if (liveNodes.isEmpty()) {
return null;
@@ -168,15 +180,20 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// todo: maybe hook up DistribStateManager /live_nodes ?
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
- public boolean simAddNode(String nodeId) throws Exception {
+
+ /**
+ * Add a new node to the cluster.
+ * @param nodeId unique node id
+ */
+ public void simAddNode(String nodeId) throws Exception {
if (liveNodes.contains(nodeId)) {
throw new Exception("Node " + nodeId + " already exists");
}
liveNodes.add(nodeId);
- return nodeReplicaMap.putIfAbsent(nodeId, new ArrayList<>()) == null;
+ nodeReplicaMap.putIfAbsent(nodeId, new ArrayList<>());
}
- // utility to run leader election in a separate thread and with throttling
+ // utility class to run leader election in a separate thread and with throttling
// Note: leader election is a no-op if a shard leader already exists for each shard
private class LeaderElection implements Callable<Boolean> {
Collection<String> collections;
@@ -202,6 +219,13 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// todo: maybe hook up DistribStateManager /live_nodes ?
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
+
+ /**
+ * Remove node from a cluster. This is equivalent to a situation when a node is lost.
+ * All replicas that were assigned to this node are marked as DOWN.
+ * @param nodeId node id
+ * @return true if a node existed and was removed
+ */
public boolean simRemoveNode(String nodeId) throws Exception {
lock.lock();
try {
@@ -224,6 +248,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Add a new replica. Note that if any details of a replica (node, coreNodeName, SolrCore name, etc)
+ * are missing they will be filled in using the policy framework.
+ * @param message replica details
+ * @param results result of the operation
+ */
public void simAddReplica(ZkNodeProps message, NamedList results) throws Exception {
AtomicLong policyVersionAfter = new AtomicLong(-1);
ClusterState clusterState = getClusterState();
@@ -245,6 +275,14 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
+
+ /**
+ * Add a replica. Note that all details of the replica must be present here, including
+ * node, coreNodeName and SolrCore name.
+ * @param nodeId node id where the replica will be added
+ * @param replicaInfo replica info
+ * @param runLeaderElection if true then run a leader election after adding the replica.
+ */
public void simAddReplica(String nodeId, ReplicaInfo replicaInfo, boolean runLeaderElection) throws Exception {
// make sure coreNodeName is unique across cluster
for (Map.Entry<String, List<ReplicaInfo>> e : nodeReplicaMap.entrySet()) {
@@ -306,6 +344,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
+
+ /**
+ * Remove replica.
+ * @param nodeId node id
+ * @param coreNodeName coreNodeName
+ */
public void simRemoveReplica(String nodeId, String coreNodeName) throws Exception {
List<ReplicaInfo> replicas = nodeReplicaMap.computeIfAbsent(nodeId, n -> new ArrayList<>());
lock.lock();
@@ -334,6 +378,10 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Save clusterstate.json to {@link DistribStateManager}.
+ * @return saved state
+ */
private ClusterState saveClusterState() throws IOException {
collectionsStatesRef.set(null);
ClusterState currentState = getClusterState();
@@ -352,6 +400,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
return currentState;
}
+ /**
+ * Delay an operation by a configured amount.
+ * @param collection collection name
+ * @param op operation name.
+ */
private void opDelay(String collection, String op) throws InterruptedException {
Map<String, Long> delays = opDelays.get(collection);
if (delays == null || delays.isEmpty() || !delays.containsKey(op)) {
@@ -360,6 +413,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
cloudManager.getTimeSource().sleep(delays.get(op));
}
+ /**
+ * Simulate running a shard leader election. This operation is a no-op if a leader already exists.
+ * If a new leader is elected the cluster state is saved.
+ * @param collections list of affected collections
+ * @param saveClusterState if true then save cluster state regardless of changes.
+ */
private synchronized void simRunLeaderElection(Collection<String> collections, boolean saveClusterState) throws Exception {
ClusterState state = getClusterState();
AtomicBoolean stateChanged = new AtomicBoolean(Boolean.FALSE);
@@ -433,6 +492,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Create a new collection. This operation uses policy framework for node and replica assignments.
+ * @param props collection details
+ * @param results results of the operation.
+ */
public void simCreateCollection(ZkNodeProps props, NamedList results) throws Exception {
if (props.getStr(CommonAdminParams.ASYNC) != null) {
results.add(CoreAdminParams.REQUESTID, props.getStr(CommonAdminParams.ASYNC));
@@ -488,6 +552,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
results.add("success", "");
}
+ /**
+ * Delete a collection
+ * @param collection collection name
+ * @param async async id
+ * @param results results of the operation
+ */
public void simDeleteCollection(String collection, String async, NamedList results) throws IOException {
if (async != null) {
results.add(CoreAdminParams.REQUESTID, async);
@@ -525,6 +595,9 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Remove all collections.
+ */
public void simDeleteAllCollections() throws Exception {
lock.lock();
try {
@@ -540,6 +613,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Move replica. This uses a similar algorithm as {@link org.apache.solr.cloud.MoveReplicaCmd#moveNormalReplica(ClusterState, NamedList, String, String, DocCollection, Replica, Slice, int, boolean)}.
+ * @param message operation details
+ * @param results operation results.
+ */
public void simMoveReplica(ZkNodeProps message, NamedList results) throws Exception {
if (message.getStr(CommonAdminParams.ASYNC) != null) {
results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC));
@@ -582,6 +660,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
results.add("success", "");
}
+ /**
+ * Create a new shard. This uses a similar algorithm as {@link CreateShardCmd}.
+ * @param message operation details
+ * @param results operation results
+ */
public void simCreateShard(ZkNodeProps message, NamedList results) throws Exception {
if (message.getStr(CommonAdminParams.ASYNC) != null) {
results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC));
@@ -638,6 +721,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Split a shard. This uses a similar algorithm as {@link SplitShardCmd}, including simulating its
+ * quirks, and leaving the original parent slice in place.
+ * @param message operation details
+ * @param results operation results.
+ */
public void simSplitShard(ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
AtomicReference<String> sliceName = new AtomicReference<>();
@@ -696,6 +785,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
+ /**
+ * Delete a shard. This uses a similar algorithm as {@link org.apache.solr.cloud.DeleteShardCmd}
+ * @param message operation details
+ * @param results operation results
+ */
public void simDeleteShard(ZkNodeProps message, NamedList results) throws Exception {
String collectionName = message.getStr(COLLECTION_PROP);
String sliceName = message.getStr(SHARD_ID_PROP);
@@ -731,7 +825,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
- public synchronized Map<String, Object> saveClusterProperties() throws Exception {
+ /**
+ * Saves cluster properties to clusterprops.json.
+ * @return current properties
+ */
+ private synchronized Map<String, Object> saveClusterProperties() throws Exception {
if (lastSavedProperties != null && lastSavedProperties.equals(clusterProperties)) {
return lastSavedProperties;
}
@@ -743,6 +841,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
return lastSavedProperties;
}
+ /**
+ * Set all cluster properties. This also updates the clusterprops.json data in
+ * {@link DistribStateManager}
+ * @param properties properties to set
+ */
public void simSetClusterProperties(Map<String, Object> properties) throws Exception {
lock.lock();
try {
@@ -756,6 +859,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Set a cluster property. This also updates the clusterprops.json data in
+ * {@link DistribStateManager}
+ * @param key property name
+ * @param value property value
+ */
public void simSetClusterProperty(String key, Object value) throws Exception {
lock.lock();
try {
@@ -770,6 +879,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Set collection properties.
+ * @param coll collection name
+ * @param properties properties
+ */
public void simSetCollectionProperties(String coll, Map<String, Object> properties) throws Exception {
if (properties == null) {
collProperties.remove(coll);
@@ -787,6 +901,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Set collection property.
+ * @param coll collection name
+ * @param key property name
+ * @param value property value
+ */
public void simSetCollectionProperty(String coll, String key, String value) throws Exception {
Map<String, Object> props = collProperties.computeIfAbsent(coll, c -> new HashMap<>());
if (value == null) {
@@ -797,6 +917,12 @@ public class SimClusterStateProvider implements ClusterStateProvider {
saveClusterState();
}
+ /**
+ * Set slice properties.
+ * @param coll collection name
+ * @param slice slice name
+ * @param properties slice properties
+ */
public void simSetSliceProperties(String coll, String slice, Map<String, Object> properties) throws Exception {
Map<String, Object> sliceProps = sliceProperties.computeIfAbsent(coll, c -> new HashMap<>()).computeIfAbsent(slice, s -> new HashMap<>());
lock.lock();
@@ -811,11 +937,56 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
+ /**
+ * Set per-collection value (eg. a metric). This value will be applied to each replica.
+ * @param collection collection name
+ * @param key property name
+ * @param value property value
+ */
+ public void simSetCollectionValue(String collection, String key, Object value) throws Exception {
+ simSetCollectionValue(collection, key, value, false);
+ }
+
+ /**
+ * Set per-collection value (eg. a metric). This value will be applied to each replica.
+ * @param collection collection name
+ * @param key property name
+ * @param value property value
+ * @param divide if the value is a {@link Number} and this is true, then the value will be evenly
+ * divided by the number of replicas.
+ */
public void simSetCollectionValue(String collection, String key, Object value, boolean divide) throws Exception {
+ simSetShardValue(collection, null, key, value, divide);
+ }
+
+ /**
+ * Set per-collection value (eg. a metric). This value will be applied to each replica in a selected shard.
+ * @param collection collection name
+ * @param shard shard name. If null then all shards will be affected.
+ * @param key property name
+ * @param value property value
+ */
+ public void simSetShardValue(String collection, String shard, String key, Object value) throws Exception {
+ simSetShardValue(collection, shard, key, value, false);
+ }
+
+ /**
+ * Set per-collection value (eg. a metric). This value will be applied to each replica in a selected shard.
+ * @param collection collection name
+ * @param shard shard name. If null then all shards will be affected.
+ * @param key property name
+ * @param value property value
+ * @param divide if the value is a {@link Number} and this is true, then the value will be evenly
+ * divided by the number of replicas.
+ */
+ public void simSetShardValue(String collection, String shard, String key, Object value, boolean divide) throws Exception {
List<ReplicaInfo> infos = new ArrayList<>();
nodeReplicaMap.forEach((n, replicas) -> {
replicas.forEach(r -> {
if (r.getCollection().equals(collection)) {
+ if (shard != null && !shard.equals(r.getShard())) {
+ return;
+ }
infos.add(r);
}
});
@@ -827,23 +998,29 @@ public class SimClusterStateProvider implements ClusterStateProvider {
value = ((Number)value).doubleValue() / infos.size();
}
for (ReplicaInfo r : infos) {
- if (value == null) {
- r.getVariables().remove(key);
- } else {
- r.getVariables().put(key, value);
+ synchronized (r) {
+ if (value == null) {
+ r.getVariables().remove(key);
+ } else {
+ r.getVariables().put(key, value);
+ }
}
}
}
- public void simSetCollectionValue(String collection, String key, Object value) throws Exception {
- simSetCollectionValue(collection, key, value, false);
- }
-
-
+ /**
+ * Return all replica infos for a node.
+ * @param node node id
+ * @return list of replicas on that node
+ */
public List<ReplicaInfo> simGetReplicaInfos(String node) {
return nodeReplicaMap.get(node);
}
+ /**
+ * List collections.
+ * @return list of existing collections.
+ */
public List<String> simListCollections() {
final Set<String> collections = new HashSet<>();
lock.lock();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
index ea00e6c..e9616f0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
@@ -46,7 +46,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Simulated {@link DistributedQueueFactory} that keeps all data in memory.
+ * Simulated {@link DistributedQueueFactory} that keeps all data in memory. Unlike
+ * the {@link GenericDistributedQueueFactory} this queue implementation data is not
+ * exposed anywhere.
*/
public class SimDistributedQueueFactory implements DistributedQueueFactory {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
index 577c976..65fb5b9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
@@ -61,6 +61,13 @@ public class SimNodeStateProvider implements NodeStateProvider {
}
// -------- simulator setup methods ------------
+
+ /**
+ * Get a node value
+ * @param node node id
+ * @param key property name
+ * @return property value or null if property or node doesn't exist.
+ */
public Object simGetNodeValue(String node, String key) {
Map<String, Object> values = nodeValues.get(node);
if (values == null) {
@@ -69,6 +76,12 @@ public class SimNodeStateProvider implements NodeStateProvider {
return values.get(key);
}
+ /**
+ * Set node values.
+ * NOTE: if values contain 'nodeRole' key then /roles.json is updated.
+ * @param node node id
+ * @param values values.
+ */
public void simSetNodeValues(String node, Map<String, Object> values) {
Map<String, Object> existing = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>());
existing.clear();
@@ -80,6 +93,13 @@ public class SimNodeStateProvider implements NodeStateProvider {
}
}
+ /**
+ * Set a node value, replacing any previous value.
+ * NOTE: if key is 'nodeRole' then /roles.json is updated.
+ * @param node node id
+ * @param key property name
+ * @param value property value
+ */
public void simSetNodeValue(String node, String key, Object value) {
Map<String, Object> existing = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>());
if (value == null) {
@@ -92,6 +112,13 @@ public class SimNodeStateProvider implements NodeStateProvider {
}
}
+ /**
+ * Add a node value, creating a list of values if necessary.
+ * NOTE: if key is 'nodeRole' then /roles.json is updated.
+ * @param node node id
+ * @param key property name
+ * @param value property value.
+ */
public void simAddNodeValue(String node, String key, Object value) {
Map<String, Object> values = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>());
Object existing = values.get(key);
@@ -110,6 +137,11 @@ public class SimNodeStateProvider implements NodeStateProvider {
}
}
+ /**
+ * Remove node values. If values contained a 'nodeRole' key then
+ * /roles.json is updated.
+ * @param node node id
+ */
public void simRemoveNodeValues(String node) {
Map<String, Object> values = nodeValues.remove(node);
if (values != null && values.containsKey("nodeRole")) {
@@ -117,6 +149,9 @@ public class SimNodeStateProvider implements NodeStateProvider {
}
}
+ /**
+ * Get all node values.
+ */
public Map<String, Map<String, Object>> simGetAllNodeValues() {
return nodeValues;
}
@@ -136,6 +171,14 @@ public class SimNodeStateProvider implements NodeStateProvider {
}
}
+ /**
+ * Simulate getting replica metrics values. This uses per-replica properties set in
+ * {@link SimClusterStateProvider#simSetCollectionValue(String, String, Object, boolean)} and
+ * similar methods.
+ * @param node node id
+ * @param tags metrics names
+ * @return map of metrics names / values
+ */
public Map<String, Object> getReplicaMetricsValues(String node, Collection<String> tags) {
List<ReplicaInfo> replicas = clusterStateProvider.simGetReplicaInfos(node);
if (replicas == null || replicas.isEmpty()) {
@@ -182,7 +225,7 @@ public class SimNodeStateProvider implements NodeStateProvider {
@Override
public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- LOG.debug("-- requested values for " + node + ": " + tags);
+ LOG.trace("-- requested values for " + node + ": " + tags);
if (!liveNodes.contains(node)) {
nodeValues.remove(node);
return Collections.emptyMap();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
index 726f839..92e3a95 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
@@ -21,9 +21,12 @@ import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -65,6 +68,7 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
static AtomicInteger triggerFiredCount = new AtomicInteger();
+ static CountDownLatch triggerFiredLatch;
static int waitForSeconds;
@BeforeClass
@@ -82,6 +86,7 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
waitForSeconds = 1 + random().nextInt(3);
triggerFiredCount.set(0);
+ triggerFiredLatch = new CountDownLatch(1);
listenerEvents.clear();
// clear any events or markers
removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
@@ -113,12 +118,12 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
@Override
public void process(TriggerEvent event, ActionContext context) throws Exception {
triggerFiredCount.incrementAndGet();
+ triggerFiredLatch.countDown();
}
}
@Test
public void testBasic() throws Exception {
- assertEquals(NUM_NODES, cluster.getClusterStateProvider().getLiveNodes().size());
SolrClient solrClient = cluster.simGetSolrClient();
String setTriggerCommand = "{" +
"'set-trigger' : {" +
@@ -179,11 +184,64 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
}
log.info("Ready after " + waitForState(collectionName, 90 * KILL_NODES, TimeUnit.SECONDS, clusterShape(2, 15)) + "ms");
- log.info(cluster.simGetSystemCollection().toString());
+
}
@Test
public void testSearchRate() throws Exception {
+ SolrClient solrClient = cluster.simGetSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'search_rate_trigger'," +
+ "'event' : 'searchRate'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'rate' : 1.0," +
+ "'enabled' : true," +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
+ "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ String setListenerCommand1 = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'srt'," +
+ "'trigger' : 'search_rate_trigger'," +
+ "'stage' : ['FAILED','SUCCEEDED']," +
+ "'class' : '" + TestTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String collectionName = "testSearchRate";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
+ "conf", 2, 10);
+ create.process(solrClient);
+ log.info("Ready after " + waitForState(collectionName, 300, TimeUnit.SECONDS, clusterShape(2, 10)) + " ms");
+
+ // collect the node names
+ Set<String> nodes = new HashSet<>();
+ cluster.getSimClusterStateProvider().getClusterState().getCollection(collectionName)
+ .getReplicas()
+ .forEach(r -> nodes.add(r.getNodeName()));
+
+ String metricName = "QUERY./select.requestTimes:1minRate";
+ // simulate search traffic
+ cluster.getSimClusterStateProvider().simSetShardValue(collectionName, "shard1", metricName, 40, true);
+
+ Thread.sleep(1000000000);
+// boolean await = triggerFiredLatch.await(20000 / SPEED, TimeUnit.MILLISECONDS);
+// assertTrue("The trigger did not fire at all", await);
+ // wait for listener to capture the SUCCEEDED stage
+ cluster.getTimeSource().sleep(2000);
+ assertEquals(listenerEvents.toString(), 1, listenerEvents.get("srt").size());
+ CapturedEvent ev = listenerEvents.get("srt").get(0);
}
}