You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/04/18 08:14:57 UTC
lucene-solr:feature/autoscaling: SOLR-10278: refactored suggester API
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 97e2607a6 -> bb48de8b0
SOLR-10278: refactored suggester API
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bb48de8b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bb48de8b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bb48de8b
Branch: refs/heads/feature/autoscaling
Commit: bb48de8b000a356f6605f9e5139df5ba7dfe1924
Parents: 97e2607
Author: Noble Paul <no...@apache.org>
Authored: Tue Apr 18 17:44:47 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Tue Apr 18 17:44:47 2017 +0930
----------------------------------------------------------------------
.../cloud/OverseerCollectionMessageHandler.java | 4 +-
.../src/java/org/apache/solr/recipe/Policy.java | 34 +--------
.../org/apache/solr/recipe/PolicyHelper.java | 74 ++++++++++++++++++++
.../test/org/apache/solr/recipe/TestPolicy.java | 13 ++--
4 files changed, 86 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bb48de8b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 71af139..b434991 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -49,7 +49,6 @@ import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.recipe.Policy;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -77,6 +76,7 @@ import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
+import org.apache.solr.recipe.PolicyHelper;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.TimeOut;
@@ -727,7 +727,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
.withClusterStateProvider(new ZkClientClusterStateProvider(zkStateReader))
.build()) {
ClientDataProvider clientDataProvider = new ClientDataProvider(csc);
- Map<String, List<String>> locations = Policy.getReplicaLocations(collName,
+ Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(collName,
zkStateReader.getZkClient().getJson(SOLR_AUTOSCALING_CONF_PATH, true),
policyName, clientDataProvider, shardNames, repFactor);
Map<Position, String> result = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bb48de8b/solr/solrj/src/java/org/apache/solr/recipe/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/recipe/Policy.java b/solr/solrj/src/java/org/apache/solr/recipe/Policy.java
index 5a8857d..e641da5 100644
--- a/solr/solrj/src/java/org/apache/solr/recipe/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/recipe/Policy.java
@@ -281,45 +281,13 @@ public class Policy implements MapWriter {
}
- enum Hint {
+ public enum Hint {
COLL, SHARD, SRC_NODE, TARGET_NODE
}
}
- public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
- String policyName, ClusterDataProvider cdp,
- List<String> shardNames,
- int repFactor) {
- Map<String, List<String>> positionMapping = new HashMap<>();
- for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
- Map policyJson = (Map) Utils.getObjectByPath(autoScalingJson, false, asList("policies", policyName));
- if (policyJson == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such policy exists " + policyName);
- }
- Map defaultPolicy = (Map) Utils.getObjectByPath(autoScalingJson, false, asList("policies", "default"));
-
- Map<String, Object> merged = Policy.mergePolicies(collName, policyJson, defaultPolicy);
- Policy policy = new Policy(merged);
- Policy.Session session = policy.createSession(cdp);
- for (String shardName : shardNames) {
- for (int i = 0; i < repFactor; i++) {
- Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
- .hint(COLL, collName)
- .hint(SHARD, shardName);
- Map op = suggester.getOperation();
- if (op == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(policy));
- }
- session = suggester.getSession();
- positionMapping.get(shardName).add((String) op.get(CoreAdminParams.NODE));
- }
- }
-
- return positionMapping;
- }
-
public static Map<String, Object> mergePolicies(String coll,
Map<String, Object> collPolicy,
Map<String, Object> defaultPolicy) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bb48de8b/solr/solrj/src/java/org/apache/solr/recipe/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/recipe/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/recipe/PolicyHelper.java
new file mode 100644
index 0000000..eea0d16
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/recipe/PolicyHelper.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.Utils;
+
+import static java.util.Arrays.asList;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.recipe.Policy.Suggester.Hint.COLL;
+import static org.apache.solr.recipe.Policy.Suggester.Hint.SHARD;
+
+public class PolicyHelper {
+ public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
+ String policyName, ClusterDataProvider cdp,
+ List<String> shardNames,
+ int repFactor) {
+ Map<String, List<String>> positionMapping = new HashMap<>();
+ for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
+ Map policyJson = (Map) Utils.getObjectByPath(autoScalingJson, false, asList("policies", policyName));
+ if (policyJson == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such policy exists " + policyName);
+ }
+ Map defaultPolicy = (Map) Utils.getObjectByPath(autoScalingJson, false, asList("policies", "default"));
+
+ Map<String, Object> merged = Policy.mergePolicies(collName, policyJson, defaultPolicy);
+ Policy policy = new Policy(merged);
+ Policy.Session session = policy.createSession(cdp);
+ for (String shardName : shardNames) {
+ for (int i = 0; i < repFactor; i++) {
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+ .hint(COLL, collName)
+ .hint(SHARD, shardName);
+ Map op = suggester.getOperation();
+ if (op == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(policy));
+ }
+ session = suggester.getSession();
+ positionMapping.get(shardName).add((String) op.get(CoreAdminParams.NODE));
+ }
+ }
+
+ return positionMapping;
+ }
+
+ public List<Map> addNode(Map<String, Object> autoScalingJson, String node, ClusterDataProvider cdp){
+ return null;
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bb48de8b/solr/solrj/src/test/org/apache/solr/recipe/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/recipe/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/recipe/TestPolicy.java
index ba9081d..3b07bf6 100644
--- a/solr/solrj/src/test/org/apache/solr/recipe/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/recipe/TestPolicy.java
@@ -171,20 +171,19 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
-/* public void testOtherTag(){
+ public void testOtherTag(){
String rules = "{" +
"conditions:[" +
"{nodeRole:'!overseer', strict:false}," +
"{replica:'<1',node:node3}," +
"{replica:'<2',node:'#ANY', shard:'#EACH'}," +
- "{replica:<3,shard:'#EACH', rack:'#ANY' }" +
+ "{replica:'<3',shard:'#EACH', rack:'#ANY' }" +
"]," +
" preferences:[" +
"{minimize:cores , precision:2}," +
"{maximize:freedisk, precision:50}, " +
"{minimize:heap, precision:1000}]}";
-
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heap:10480, rack: rack4}," +
"node2:{cores:4, freedisk: 749, heap:6873, rack: rack3}," +
@@ -194,7 +193,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
Policy.Session session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
- }*/
+ Map op = session
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "s1").getOperation();
+ assertNotNull(op);
+ }
+
private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
return new ClusterDataProvider() {