You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2017/04/10 06:45:18 UTC
lucene-solr:feature/autoscaling: SOLR-10374: Implement set-policy and
remove-policy APIs
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling dd0bd7d01 -> 5c85e8e59
SOLR-10374: Implement set-policy and remove-policy APIs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5c85e8e5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5c85e8e5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5c85e8e5
Branch: refs/heads/feature/autoscaling
Commit: 5c85e8e59d91f5c0b8fca50928a67de943402aae
Parents: dd0bd7d
Author: Cao Manh Dat <da...@apache.org>
Authored: Mon Apr 10 13:45:07 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Mon Apr 10 13:45:07 2017 +0700
----------------------------------------------------------------------
.../cloud/autoscaling/AutoScalingHandler.java | 70 +++++++++++++++++
.../resources/apispec/autoscaling.Commands.json | 30 +++++++
.../autoscaling/AutoScalingHandlerTest.java | 82 ++++++++++++++++++++
3 files changed, 182 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c85e8e5/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 781aad5..db415f8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -106,10 +106,56 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
case "resume-trigger":
handleResumeTrigger(req, rsp, op);
break;
+ case "set-policy":
+ handleSetPolicies(req, rsp, op);
+ break;
+ case "remove-policy":
+ handleRemovePolicy(req, rsp, op);
}
}
}
+ private void handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ String policyName = (String) op.getCommandData();
+
+ if (policyName.trim().length() == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The policy name cannot be empty");
+ }
+ Map<String, Object> autoScalingConf = zkReadAutoScalingConf(container.getZkController().getZkStateReader());
+ Map<String, Object> policies = (Map<String, Object>) autoScalingConf.get("policies");
+ if (policies == null || !policies.containsKey(policyName)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No policy exists with name: " + policyName);
+ }
+
+ zkSetPolicies(container.getZkController().getZkStateReader(), policyName, null);
+ rsp.getValues().add("result", "success");
+ }
+
+ private void handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ String policyName = op.getStr("name");
+
+ if (policyName == null || policyName.trim().length() == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The policy name cannot be null or empty");
+ }
+
+ Set<String> keys = op.getDataMap().keySet();
+ boolean isValid = false;
+ for (String key : keys) {
+ if (key.equals("conditions") || key.equals("preferences")) isValid = true;
+ else if(!key.equals("name")){
+ isValid = false;
+ break;
+ }
+ }
+ if (!isValid) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "No conditions or peferences are specified for the policy " + policyName);
+ }
+
+ zkSetPolicies(container.getZkController().getZkStateReader(), policyName, op.getValuesExcluding("name"));
+ rsp.getValues().add("result", "success");
+ }
+
private void handleResumeTrigger(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
String triggerName = op.getStr("name");
@@ -396,6 +442,30 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
+ private void zkSetPolicies(ZkStateReader reader, String policyName, Map<String, Object> policyProperties) throws KeeperException, InterruptedException {
+ while (true) {
+ Stat stat = new Stat();
+ ZkNodeProps loaded = null;
+ byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
+ loaded = ZkNodeProps.load(data);
+ Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");
+ if (policies == null) policies = new HashMap<>(1);
+ if (policyProperties != null) {
+ policies.put(policyName, policyProperties);
+ } else {
+ policies.remove(policyName);
+ }
+ loaded = loaded.plus("policies", policies);
+ try {
+ reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
+ } catch (KeeperException.BadVersionException bve) {
+ // somebody else has changed the configuration so we must retry
+ continue;
+ }
+ break;
+ }
+ }
+
private Map<String, Object> zkReadAutoScalingConf(ZkStateReader reader) throws KeeperException, InterruptedException {
byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
ZkNodeProps loaded = ZkNodeProps.load(data);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c85e8e5/solr/core/src/resources/apispec/autoscaling.Commands.json
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/apispec/autoscaling.Commands.json b/solr/core/src/resources/apispec/autoscaling.Commands.json
index 502b9c8..8e8d8ec 100644
--- a/solr/core/src/resources/apispec/autoscaling.Commands.json
+++ b/solr/core/src/resources/apispec/autoscaling.Commands.json
@@ -179,6 +179,36 @@
"required": [
"name"
]
+ },
+ "set-policy" : {
+ "type":"object",
+ "description": "The set-policy command allows you to add and update policies",
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the policy"
+ },
+ "conditions" : {
+ "type": "array",
+ "description": "Conditions of the policy"
+ },
+ "preferences": {
+ "type": "array",
+ "description": "Preferences of the policy"
+ }
+ },
+ "oneOf": [{
+ "required": ["name", "preferences"]
+ },{
+ "required": ["name", "conditions"]
+ },{
+ "required": ["name", "preferences", "conditions"]
+ }],
+ "additionalProperties": false
+ },
+ "remove-policy": {
+ "description": "The remove-policy command allows you to remove a policy",
+ "type": "string"
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c85e8e5/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index dd9a3d0..f2b8876 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -345,6 +345,88 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
} catch (HttpSolrClient.RemoteSolrException e) {
// expected
}
+
+ // add multiple poilicies
+ String setPolicyCommand = "{\n" +
+ "\t\"set-policy\": {\n" +
+ "\t\t\"name\" : \"default\",\n" +
+ "\t\t\"preferences\": [\n" +
+ "\t\t\t{\n" +
+ "\t\t\t\t\"minimize\": \"replicas\",\n" +
+ "\t\t\t\t\"precision\": 3\n" +
+ "\t\t\t},\n" +
+ "\t\t\t{\n" +
+ "\t\t\t\t\"maximize\": \"freedisk\",\n" +
+ "\t\t\t\t\"precision\": 100\n" +
+ "\t\t\t}\n" +
+ "\t\t]\t\t\n" +
+ "\t}, \n" +
+ "\t\"set-policy\": {\n" +
+ "\t\t\"name\" : \"policy1\",\n" +
+ "\t\t\"preferences\": [\n" +
+ "\t\t\t{\n" +
+ "\t\t\t\t\"minimize\": \"cpu\",\n" +
+ "\t\t\t\t\"precision\": 10\n" +
+ "\t\t\t}\n" +
+ "\t\t]\n" +
+ "\t}\n" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");
+ assertNotNull(policies);
+ assertNotNull(policies.get("default"));
+ assertNotNull(policies.get("policy1"));
+
+ // update default policy
+ setPolicyCommand = "{\n" +
+ "\t\"set-policy\": {\n" +
+ "\t\t\"name\" : \"default\",\n" +
+ "\t\t\"preferences\": [\n" +
+ "\t\t\t{\n" +
+ "\t\t\t\t\"minimize\": \"replicas\",\n" +
+ "\t\t\t\t\"precision\": 3\n" +
+ "\t\t\t}\n" +
+ "\t\t]\t\t\n" +
+ "\t}\n" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ policies = (Map<String, Object>) loaded.get("policies");
+ Map<String,Object> properties = (Map<String, Object>) policies.get("default");
+ List preferences = (List) properties.get("preferences");
+ assertEquals(1, preferences.size());
+
+ // policy is not valid
+ setPolicyCommand = "{\n" +
+ "\t\"set-policy\": {\n" +
+ "\t\t\"name\" : \"default\"\t\n" +
+ "\t}\n" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
+ try {
+ response = solrClient.request(req);
+ fail("Adding a policy without conditions or preferences should have failed");
+ } catch (HttpSolrClient.RemoteSolrException e) {
+ // expected
+ }
+
+ String removePolicyCommand = "{\n" +
+ "\t\"remove-policy\" : \"policy1\"\n" +
+ "}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, removePolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ policies = (Map<String, Object>) loaded.get("policies");
+ assertNull(policies.get("policy1"));
}
static class AutoScalingRequest extends SolrRequest {