You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/05/02 07:12:09 UTC
lucene-solr:feature/autoscaling: SOLR-10374: Added set-cluster-policy,
set-cluster-preferences commands
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 70462ed6a -> 3cf4b9272
SOLR-10374: Added set-cluster-policy, set-cluster-preferences commands
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3cf4b927
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3cf4b927
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3cf4b927
Branch: refs/heads/feature/autoscaling
Commit: 3cf4b92725a0949cce536502ee68e10182509436
Parents: 70462ed
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue May 2 12:42:13 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue May 2 12:42:13 2017 +0530
----------------------------------------------------------------------
.../cloud/autoscaling/AutoScalingHandler.java | 99 +++++++++++++++-----
.../org/apache/solr/util/CommandOperation.java | 26 ++++-
.../resources/apispec/autoscaling.Commands.json | 45 ++++-----
.../autoscaling/AutoScalingHandlerTest.java | 98 ++++++++++++-------
4 files changed, 184 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3cf4b927/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 6922c79..2ac1af5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.ImmutableSet;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.common.SolrException;
@@ -59,6 +60,8 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected final CoreContainer container;
private final List<Map<String, String>> DEFAULT_ACTIONS = new ArrayList<>(3);
+ private static ImmutableSet<String> singletonCommands = ImmutableSet.of("set-cluster-preferences", "set-cluster-policy");
+
public AutoScalingHandler(CoreContainer container) {
this.container = container;
@@ -81,7 +84,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
if (req.getContentStreams() == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No contentStream");
}
- List<CommandOperation> ops = CommandOperation.readCommands(req.getContentStreams(), rsp);
+ List<CommandOperation> ops = CommandOperation.readCommands(req.getContentStreams(), rsp, singletonCommands);
if (ops == null) {
// errors have already been added to the response so there's nothing left to do
return;
@@ -111,10 +114,37 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
break;
case "remove-policy":
handleRemovePolicy(req, rsp, op);
+ break;
+ case "set-cluster-preferences":
+ handleSetClusterPreferences(req, rsp, op);
+ break;
+ case "set-cluster-policy":
+ handleSetClusterPolicy(req, rsp, op);
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown command: " + op.name);
}
}
}
+ private void handleSetClusterPolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ List clusterPolicy = (List) op.getCommandData();
+ if (clusterPolicy == null || !(clusterPolicy instanceof List)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "A list of cluster policies was not found");
+ }
+ zkSetClusterPolicy(container.getZkController().getZkStateReader(), clusterPolicy);
+ rsp.getValues().add("result", "success");
+ }
+
+ private void handleSetClusterPreferences(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ List preferences = (List) op.getCommandData();
+ if (preferences == null || !(preferences instanceof List)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "A list of cluster preferences not found");
+ }
+ zkSetPreferences(container.getZkController().getZkStateReader(), preferences);
+ rsp.getValues().add("result", "success");
+ }
+
private void handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
String policyName = (String) op.getCommandData();
@@ -132,27 +162,16 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
private void handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
- String policyName = op.getStr("name");
-
- if (policyName == null || policyName.trim().length() == 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The policy name cannot be null or empty");
- }
-
- Set<String> keys = op.getDataMap().keySet();
- boolean isValid = false;
- for (String key : keys) {
- if (key.equals("conditions") || key.equals("preferences")) isValid = true;
- else if(!key.equals("name")){
- isValid = false;
- break;
+ Map<String, Object> policies = op.getDataMap();
+ for (Map.Entry<String, Object> policy: policies.entrySet()) {
+ String policyName = policy.getKey();
+ if (policyName == null || policyName.trim().length() == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The policy name cannot be null or empty");
}
}
- if (!isValid) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "No conditions or peferences are specified for the policy " + policyName);
- }
- zkSetPolicies(container.getZkController().getZkStateReader(), policyName, op.getValuesExcluding("name"));
+ zkSetPolicies(container.getZkController().getZkStateReader(), null, policies);
+
rsp.getValues().add("result", "success");
}
@@ -442,7 +461,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
- private void zkSetPolicies(ZkStateReader reader, String policyName, Map<String, Object> policyProperties) throws KeeperException, InterruptedException {
+ private void zkSetPolicies(ZkStateReader reader, String policyBeRemoved, Map<String, Object> newPolicies) throws KeeperException, InterruptedException {
while (true) {
Stat stat = new Stat();
ZkNodeProps loaded = null;
@@ -450,10 +469,10 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
loaded = ZkNodeProps.load(data);
Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");
if (policies == null) policies = new HashMap<>(1);
- if (policyProperties != null) {
- policies.put(policyName, policyProperties);
+ if (newPolicies != null) {
+ policies.putAll(newPolicies);
} else {
- policies.remove(policyName);
+ policies.remove(policyBeRemoved);
}
loaded = loaded.plus("policies", policies);
try {
@@ -466,6 +485,40 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
+ private void zkSetPreferences(ZkStateReader reader, List preferences) throws KeeperException, InterruptedException {
+ while (true) {
+ Stat stat = new Stat();
+ ZkNodeProps loaded = null;
+ byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
+ loaded = ZkNodeProps.load(data);
+ loaded = loaded.plus("cluster-preferences", preferences);
+ try {
+ reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
+ } catch (KeeperException.BadVersionException bve) {
+ // somebody else has changed the configuration so we must retry
+ continue;
+ }
+ break;
+ }
+ }
+
+ private void zkSetClusterPolicy(ZkStateReader reader, List clusterPolicy) throws KeeperException, InterruptedException {
+ while (true) {
+ Stat stat = new Stat();
+ ZkNodeProps loaded = null;
+ byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
+ loaded = ZkNodeProps.load(data);
+ loaded = loaded.plus("cluster-policy", clusterPolicy);
+ try {
+ reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
+ } catch (KeeperException.BadVersionException bve) {
+ // somebody else has changed the configuration so we must retry
+ continue;
+ }
+ break;
+ }
+ }
+
private Map<String, Object> zkReadAutoScalingConf(ZkStateReader reader) throws KeeperException, InterruptedException {
byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
ZkNodeProps loaded = ZkNodeProps.load(data);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3cf4b927/solr/core/src/java/org/apache/solr/util/CommandOperation.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/CommandOperation.java b/solr/core/src/java/org/apache/solr/util/CommandOperation.java
index 88dfbb9..9e9a5c2 100644
--- a/solr/core/src/java/org/apache/solr/util/CommandOperation.java
+++ b/solr/core/src/java/org/apache/solr/util/CommandOperation.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
@@ -218,11 +219,17 @@ public class CommandOperation {
return errors;
}
+ public static List<CommandOperation> parse(Reader rdr) throws IOException {
+ return parse(rdr, Collections.emptySet());
+ }
/**
* Parse the command operations into command objects
+ * @param rdr The payload
+ * @param singletonCommands commands that cannot be repeated
+ * @return parsed list of commands
*/
- public static List<CommandOperation> parse(Reader rdr) throws IOException {
+ public static List<CommandOperation> parse(Reader rdr, Set<String> singletonCommands) throws IOException {
JSONParser parser = new JSONParser(rdr);
ObjectBuilder ob = new ObjectBuilder(parser);
@@ -237,7 +244,7 @@ public class CommandOperation {
Object key = ob.getKey();
ev = parser.nextEvent();
Object val = ob.getVal();
- if (val instanceof List) {
+ if (val instanceof List && !singletonCommands.contains(key)) {
List list = (List) val;
for (Object o : list) {
if (!(o instanceof Map)) {
@@ -280,7 +287,18 @@ public class CommandOperation {
}
}
- public static List<CommandOperation> readCommands(Iterable<ContentStream> streams, SolrQueryResponse resp)
+ public static List<CommandOperation> readCommands(Iterable<ContentStream> streams, SolrQueryResponse resp) throws IOException {
+ return readCommands(streams, resp, Collections.emptySet());
+ }
+
+ /**Read commands from request streams
+ * @param streams the streams
+ * @param resp solr query response
+ * @param singletonCommands , commands that cannot be repeated
+ * @return parsed list of commands
+ * @throws IOException
+ */
+ public static List<CommandOperation> readCommands(Iterable<ContentStream> streams, SolrQueryResponse resp, Set<String> singletonCommands)
throws IOException {
if (streams == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
@@ -288,7 +306,7 @@ public class CommandOperation {
ArrayList<CommandOperation> ops = new ArrayList<>();
for (ContentStream stream : streams)
- ops.addAll(parse(stream.getReader()));
+ ops.addAll(parse(stream.getReader(), singletonCommands));
List<Map> errList = CommandOperation.captureErrors(ops);
if (!errList.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errList);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3cf4b927/solr/core/src/resources/apispec/autoscaling.Commands.json
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/apispec/autoscaling.Commands.json b/solr/core/src/resources/apispec/autoscaling.Commands.json
index 8e8d8ec..1bc1295 100644
--- a/solr/core/src/resources/apispec/autoscaling.Commands.json
+++ b/solr/core/src/resources/apispec/autoscaling.Commands.json
@@ -182,33 +182,34 @@
},
"set-policy" : {
"type":"object",
- "description": "The set-policy command allows you to add and update policies",
- "properties": {
- "name": {
- "type": "string",
- "description": "The name of the policy"
- },
- "conditions" : {
- "type": "array",
- "description": "Conditions of the policy"
- },
- "preferences": {
- "type": "array",
- "description": "Preferences of the policy"
+ "description": "The set-policy command allows you to add and update policies that apply to collections",
+ "patternProperties": {
+ "^.+$": {
+ "type": "array"
}
},
- "oneOf": [{
- "required": ["name", "preferences"]
- },{
- "required": ["name", "conditions"]
- },{
- "required": ["name", "preferences", "conditions"]
- }],
"additionalProperties": false
},
+ "set-cluster-policy" : {
+ "type" : "array",
+ "description" : "The set-cluster-policy command allows you to add and update cluster-level policy that acts as the base for all collection level policies, if any"
+ },
+ "set-cluster-preferences" : {
+ "type" : "array",
+ "description" : "The set-cluster-preferences command allows you to add and update cluster-level preferences that are used to sort nodes for selection in cluster operations"
+ },
"remove-policy": {
- "description": "The remove-policy command allows you to remove a policy",
- "type": "string"
+ "description": "Remove a policy",
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the policy to be removed"
+ }
+ },
+ "required": [
+ "name"
+ ]
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3cf4b927/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 97fe100..68b86b0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -340,16 +340,16 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
}
// add multiple policies
- String setPolicyCommand = "{" +
- "'set-policy': {" +
- "'name' : 'default'," +
- "'preferences': [" +
- "{'minimize': 'replicas','precision': 3}," +
- "{'maximize': 'freedisk','precision': 100}]" +
- "}, " +
- "'set-policy': {" +
- "'name' : 'policy1'," +
- "'preferences': [{'minimize': 'cpu','precision': 10}]}}";
+ String setPolicyCommand = "{'set-policy': {" +
+ " 'xyz':[" +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'!overseer', 'replica':'#ANY'}" +
+ " ]," +
+ " 'policy1':[" +
+ " {'cores':'<2', 'node':'#ANY'}," +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
+ " ]" +
+ "}}";
req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -357,41 +357,25 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
loaded = ZkNodeProps.load(data);
Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");
assertNotNull(policies);
- assertNotNull(policies.get("default"));
+ assertNotNull(policies.get("xyz"));
assertNotNull(policies.get("policy1"));
// update default policy
- setPolicyCommand = "{" +
- "'set-policy': {" +
- "'name' : 'default'," +
- "'preferences': [" +
- "{" +
- "'minimize': 'replicas'," +
- "'precision': 3" +
- "}]}}";
+ setPolicyCommand = "{'set-policy': {" +
+ " 'xyz':[" +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
+ " ]" +
+ "}}";
req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
loaded = ZkNodeProps.load(data);
policies = (Map<String, Object>) loaded.get("policies");
- Map<String,Object> properties = (Map<String, Object>) policies.get("default");
- List preferences = (List) properties.get("preferences");
- assertEquals(1, preferences.size());
-
- // policy is not valid
- setPolicyCommand = "{" +
- "'set-policy': {" +
- "'name' : 'default'" +
- "}}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
- try {
- response = solrClient.request(req);
- fail("Adding a policy without conditions or preferences should have failed");
- } catch (HttpSolrClient.RemoteSolrException e) {
- // expected
- }
+ List conditions = (List) policies.get("xyz");
+ assertEquals(1, conditions.size());
+ // remove policy
String removePolicyCommand = "{remove-policy : policy1}";
req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, removePolicyCommand);
response = solrClient.request(req);
@@ -400,6 +384,50 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
loaded = ZkNodeProps.load(data);
policies = (Map<String, Object>) loaded.get("policies");
assertNull(policies.get("policy1"));
+
+ // set preferences
+ String setPreferencesCommand = "{" +
+ " 'set-cluster-preferences': [" +
+ " {'minimize': 'replicas', 'precision': 3}," +
+ " {'maximize': 'freedisk','precision': 100}," +
+ " {'minimize': 'cpu','precision': 10}]" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPreferencesCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ List preferences = (List) loaded.get("cluster-preferences");
+ assertEquals(3, preferences.size());
+
+ // set preferences
+ setPreferencesCommand = "{" +
+ " 'set-cluster-preferences': [" +
+ " {'minimize': 'cpu','precision': 10}]" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPreferencesCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ preferences = (List) loaded.get("cluster-preferences");
+ assertEquals(1, preferences.size());
+
+ String setClusterPolicyCommand = "{" +
+ " 'set-cluster-policy': [" +
+ " {'cores':'<10', 'node':'#ANY'}," +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'!overseer', 'replica':'#ANY'}" +
+ " ]" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setClusterPolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ List clusterPolicy = (List) loaded.get("cluster-policy");
+ assertNotNull(clusterPolicy);
+ assertEquals(3, clusterPolicy.size());
}
static class AutoScalingRequest extends SolrRequest {