You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/09/27 14:22:40 UTC
lucene-solr:jira/solr-11285-sim: SOLR-11403: some improvements to
simulator setup.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11285-sim b6afb0f45 -> d0142aaac
SOLR-11403: some improvements to simulator setup.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d0142aaa
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d0142aaa
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d0142aaa
Branch: refs/heads/jira/solr-11285-sim
Commit: d0142aaac227c20c47fa226bb68d455436250ec9
Parents: b6afb0f
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Sep 27 16:21:59 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Sep 27 16:21:59 2017 +0200
----------------------------------------------------------------------
.../solr/handler/admin/CollectionsHandler.java | 2 +-
.../autoscaling/sim/SimCloudDataProvider.java | 36 +++--
.../autoscaling/sim/SimClusterDataProvider.java | 142 ++++++++++++++++---
.../sim/TestClusterDataProvider.java | 20 +--
4 files changed, 144 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d0142aaa/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 3d01222..d14d4f3 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -388,7 +388,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
results.add("status", status);
}
- enum CollectionOperation implements CollectionOp {
+ public enum CollectionOperation implements CollectionOp {
/**
* very simple currently, you can pass a template collection, and the new collection is created on
* every node the template collection is on
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d0142aaa/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudDataProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudDataProvider.java
index c2e1589..62f6e1b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudDataProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudDataProvider.java
@@ -27,27 +27,31 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.SolrResponseBase;
/**
* Simulated {@link SolrCloudDataProvider}.
*/
public class SimCloudDataProvider implements SolrCloudDataProvider {
- private final DistribStateManager stateManager;
- private final ClusterDataProvider dataProvider;
- private final DistributedQueueFactory queueFactory;
- private final SolrClient solrClient;
+ private final SimDistribStateManager stateManager;
+ private final SimClusterDataProvider dataProvider;
+ private final SimDistributedQueueFactory queueFactory;
+ private SolrClient solrClient;
private final SimHttpServer httpServer;
- public SimCloudDataProvider(DistribStateManager stateManager, ClusterDataProvider dataProvider,
- DistributedQueueFactory queueFactory, SolrClient solrClient) {
- this.stateManager = stateManager;
- this.dataProvider = dataProvider;
- this.queueFactory = queueFactory;
- this.solrClient = solrClient;
+ public SimCloudDataProvider() {
+ this.stateManager = new SimDistribStateManager();
+ this.dataProvider = new SimClusterDataProvider();
+ this.queueFactory = new SimDistributedQueueFactory();
this.httpServer = new SimHttpServer();
}
+ public void setSolrClient(SolrClient solrClient) {
+ this.solrClient = solrClient;
+ }
+
@Override
public ClusterDataProvider getClusterDataProvider() {
return dataProvider;
@@ -65,10 +69,14 @@ public class SimCloudDataProvider implements SolrCloudDataProvider {
@Override
public SolrResponse request(SolrRequest req) throws IOException {
- try {
- return req.process(solrClient);
- } catch (SolrServerException e) {
- throw new IOException(e);
+ if (solrClient != null) {
+ try {
+ return req.process(solrClient);
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ } else {
+ return dataProvider.simHandleSolrRequest(req);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d0142aaa/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterDataProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterDataProvider.java
index 3617a750..4b7ab91 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterDataProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterDataProvider.java
@@ -18,6 +18,7 @@
package org.apache.solr.cloud.autoscaling.sim;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -27,31 +28,45 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.SolrResponseBase;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class SimClusterDataProvider implements ClusterDataProvider {
+ private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Map<String, List<ReplicaInfo>> nodes = new ConcurrentHashMap<>();
private final Map<String, Map<String, Object>> nodeValues = new ConcurrentHashMap<>();
+ private final List<Watcher> autoscalingWatchers = new ArrayList<>();
+ private final Map<String, Object> clusterProperties = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, Object>> collProperties = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, Map<String, Object>>> sliceProperties = new ConcurrentHashMap<>();
+
private AutoScalingConfig autoScalingConfig = new AutoScalingConfig(Collections.emptyMap());
- private List<Watcher> autoscalingWatchers = new ArrayList<>();
- private Map<String, Object> clusterProperties = new ConcurrentHashMap<>();
- private Map<String, Map<String, Object>> collProperties = new ConcurrentHashMap<>();
- private Map<String, Map<String, Map<String, Object>>> sliceProperties = new ConcurrentHashMap<>();
+ private SimDistribStateManager distribStateManager;
/**
* Zero-arg constructor. The instance needs to be initialized using the <code>sim*</code> methods in order
@@ -68,15 +83,22 @@ public class SimClusterDataProvider implements ClusterDataProvider {
* @param nodeValues per-node values (eg. metrics)
*/
public SimClusterDataProvider(ClusterState initialState, AutoScalingConfig autoScalingConfig, Map<String, Map<String, Object>> nodeValues) {
- Collection<String> liveNodes = initialState.getLiveNodes();
if (nodeValues != null) {
- nodeValues.entrySet().stream()
- .filter(e -> liveNodes.contains(e.getKey()))
- .forEach(e -> nodeValues.put(e.getKey(), e.getValue()));
+ this.nodeValues.putAll(nodeValues);
}
if (autoScalingConfig != null) {
this.autoScalingConfig = (AutoScalingConfig)autoScalingConfig.clone();
}
+ simSetClusterState(initialState);
+ }
+
+ // ============== SIMULATOR SETUP METHODS ====================
+
+ public void simSetClusterState(ClusterState initialState) {
+ collProperties.clear();
+ sliceProperties.clear();
+ nodes.clear();
+ Collection<String> liveNodes = initialState.getLiveNodes();
initialState.forEachCollection(dc -> {
collProperties.computeIfAbsent(dc.getName(), name -> new HashMap<>()).putAll(dc.getProperties());
dc.getSlices().forEach(s -> {
@@ -90,42 +112,56 @@ public class SimClusterDataProvider implements ClusterDataProvider {
});
});
});
+ nodeValues.keySet().retainAll(liveNodes);
}
- // ============== SIMULATOR SETUP METHODS ====================
+ public void simSetDistribStateManager(SimDistribStateManager simDistribStateManager) {
+ this.distribStateManager = simDistribStateManager;
+ }
- public void simAddNodes(Collection<String> nodeIds) {
+ public void simAddNodes(Collection<String> nodeIds) throws Exception {
+ for (String node : nodeIds) {
+ if (nodes.containsKey(node)) {
+ throw new Exception("Node " + node + " already exists");
+ }
+ }
for (String node : nodeIds) {
simAddNode(node);
}
}
+
// todo: maybe hook up DistribStateManager /live_nodes ?
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
- public void simAddNode(String nodeId) {
- nodes.putIfAbsent(nodeId, new ArrayList<>());
+ public boolean simAddNode(String nodeId) throws Exception {
+ if (nodes.containsKey(nodeId)) {
+ throw new Exception("Node " + nodeId + " already exists");
+ }
+ return nodes.putIfAbsent(nodeId, new ArrayList<>()) == null;
}
// todo: maybe hook up DistribStateManager /live_nodes ?
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
- public void simRemoveNode(String nodeId) {
+ public boolean simRemoveNode(String nodeId) {
nodes.remove(nodeId);
- nodeValues.remove(nodeId);
+ return nodeValues.remove(nodeId) != null;
}
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
- public void simAddReplica(String nodeId, ReplicaInfo replicaInfo) {
- List<ReplicaInfo> replicas = nodes.computeIfAbsent(nodeId, n -> new ArrayList<>());
- // make sure coreNodeName is unique
- for (ReplicaInfo ri : replicas) {
- if (ri.getCore().equals(replicaInfo.getCore())) {
- throw new RuntimeException("Duplicate coreNodeName for existing=" + ri + " and new=" + replicaInfo);
+ public void simAddReplica(String nodeId, ReplicaInfo replicaInfo) throws Exception {
+ // make sure coreNodeName is unique across cluster
+ for (Map.Entry<String, List<ReplicaInfo>> e : nodes.entrySet()) {
+ for (ReplicaInfo ri : e.getValue()) {
+ if (ri.getCore().equals(replicaInfo.getCore())) {
+ throw new Exception("Duplicate coreNodeName for existing=" + ri + " on node " + e.getKey() + " and new=" + replicaInfo);
+ }
}
}
+ List<ReplicaInfo> replicas = nodes.computeIfAbsent(nodeId, n -> new ArrayList<>());
replicas.add(replicaInfo);
}
// todo: maybe hook up DistribStateManager /clusterstate.json watchers?
- public void simRemoveReplica(String nodeId, String coreNodeName) throws IOException {
+ public void simRemoveReplica(String nodeId, String coreNodeName) throws Exception {
List<ReplicaInfo> replicas = nodes.computeIfAbsent(nodeId, n -> new ArrayList<>());
for (int i = 0; i < replicas.size(); i++) {
if (coreNodeName.equals(replicas.get(i).getCore())) {
@@ -133,7 +169,7 @@ public class SimClusterDataProvider implements ClusterDataProvider {
return;
}
}
- throw new IOException("Replica " + coreNodeName + " not found on node " + nodeId);
+ throw new Exception("Replica " + coreNodeName + " not found on node " + nodeId);
}
public void simSetNodeValues(String node, Map<String, Object> values) {
@@ -198,6 +234,68 @@ public class SimClusterDataProvider implements ClusterDataProvider {
autoscalingWatchers.clear();
}
+ public SolrResponse simHandleSolrRequest(SolrRequest req) throws IOException {
+ // support only a specific subset of collection admin ops
+ if (!(req instanceof CollectionAdminRequest)) {
+ throw new UnsupportedOperationException("Only CollectionAdminRequest-s are supported: " + req.getClass().getName());
+ }
+ SolrParams params = req.getParams();
+ String a = params.get(CoreAdminParams.ACTION);
+ SolrResponse rsp = new SolrResponseBase();
+ if (a != null) {
+ CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(a);
+ if (action == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + a);
+ }
+ switch (action) {
+ case ADDREPLICA:
+ break;
+ case ADDREPLICAPROP:
+ break;
+ case CREATE:
+ break;
+ case CLUSTERPROP:
+ break;
+ case CREATESHARD:
+ break;
+ case DELETE:
+ break;
+ case DELETENODE:
+ break;
+ case DELETEREPLICA:
+ break;
+ case DELETEREPLICAPROP:
+ break;
+ case DELETESHARD:
+ break;
+ case FORCELEADER:
+ break;
+ case LIST:
+ break;
+ case MODIFYCOLLECTION:
+ break;
+ case MOVEREPLICA:
+ break;
+ case REBALANCELEADERS:
+ break;
+ case RELOAD:
+ break;
+ case REPLACENODE:
+ break;
+ case SPLITSHARD:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported collection admin action=" + action);
+ }
+ LOG.info("Invoked Collection Action :{} with params {}", action.toLower(), req.getParams().toQueryString());
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "action is a required param");
+ }
+
+ return rsp;
+
+ }
+
// interface methods
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d0142aaa/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestClusterDataProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestClusterDataProvider.java
index 5089be2..5485cc6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestClusterDataProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestClusterDataProvider.java
@@ -82,28 +82,10 @@ public class TestClusterDataProvider extends SolrCloudTestCase {
// initialize simulated provider
SimClusterDataProvider simProvider = new SimClusterDataProvider();
simProvider.simSetAutoScalingConfig(autoScalingConfig);
- simProvider.simSetClusterProperties(clusterProperties);
- simProvider.simAddNodes(liveNodes);
- realState.getCollectionsMap().forEach((coll, dc) -> {
- dc.getSlices().forEach(slice -> {
- simProvider.setSliceProperties(coll, slice.getName(), slice.getProperties());
- });
- });
- replicas.forEach((n, infos) -> {
- infos.forEach((coll, shardInfos) -> {
- shardInfos.forEach((shard, replicaInfos) -> {
- replicaInfos.forEach(ri -> {
- simProvider.simAddReplica(n, ri);
- });
- });
- });
- });
nodeValues.forEach((n, values) -> {
simProvider.simSetNodeValues(n, values);
});
- realState.getCollectionsMap().forEach((name, docColl) -> {
- simProvider.simSetCollectionProperties(name, docColl.getProperties());
- });
+ simProvider.simSetClusterState(realState);
ClusterState simState = simProvider.getClusterState();
assertClusterStateEquals(realState, simState);
// test the other constructor