You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/05/29 13:33:16 UTC
lucene-solr:feature/autoscaling: SOLR-10764: AutoScalingHandler
should validate policy and preferences before updating zookeeper
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 011712ff0 -> d6b239158
SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d6b23915
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d6b23915
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d6b23915
Branch: refs/heads/feature/autoscaling
Commit: d6b23915837289ea5c6aa6fde66bf9bee6f36dee
Parents: 011712f
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon May 29 19:03:09 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon May 29 19:03:09 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../cloud/autoscaling/AutoScalingHandler.java | 33 +++++++++++++++-----
.../autoscaling/AutoScalingHandlerTest.java | 25 ++++++++++++++-
3 files changed, 51 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d6b23915/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5166da7..1ca238d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -142,6 +142,8 @@ Other Changes
* SOLR-10643: Throttling strategy for triggers and policy executions. (shalin)
+* SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper. (shalin)
+
----------------------
================== 6.6.0 ==================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d6b23915/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index cd18307..4d3c566 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -143,7 +143,11 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
}
- } finally {
+ } catch (Exception e) {
+ rsp.getValues().add("result", "failure");
+ throw e;
+ }
+ finally {
RequestHandlerUtils.addExperimentalFormatWarning(rsp);
}
}
@@ -182,7 +186,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
- private void handleSetClusterPolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ private void handleSetClusterPolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
List clusterPolicy = (List) op.getCommandData();
if (clusterPolicy == null || !(clusterPolicy instanceof List)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "A list of cluster policies was not found");
@@ -191,7 +195,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
rsp.getValues().add("result", "success");
}
- private void handleSetClusterPreferences(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ private void handleSetClusterPreferences(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
List preferences = (List) op.getCommandData();
if (preferences == null || !(preferences instanceof List)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "A list of cluster preferences not found");
@@ -200,7 +204,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
rsp.getValues().add("result", "success");
}
- private void handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ private void handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
String policyName = (String) op.getCommandData();
if (policyName.trim().length() == 0) {
@@ -216,7 +220,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
rsp.getValues().add("result", "success");
}
- private void handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+ private void handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
Map<String, Object> policies = op.getDataMap();
for (Map.Entry<String, Object> policy: policies.entrySet()) {
String policyName = policy.getKey();
@@ -516,7 +520,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
- private void zkSetPolicies(ZkStateReader reader, String policyBeRemoved, Map<String, Object> newPolicies) throws KeeperException, InterruptedException {
+ private void zkSetPolicies(ZkStateReader reader, String policyBeRemoved, Map<String, Object> newPolicies) throws KeeperException, InterruptedException, IOException {
while (true) {
Stat stat = new Stat();
ZkNodeProps loaded = null;
@@ -530,6 +534,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
policies.remove(policyBeRemoved);
}
loaded = loaded.plus("policies", policies);
+ verifyAutoScalingConf(loaded.getProperties());
try {
reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
} catch (KeeperException.BadVersionException bve) {
@@ -540,13 +545,14 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
- private void zkSetPreferences(ZkStateReader reader, List preferences) throws KeeperException, InterruptedException {
+ private void zkSetPreferences(ZkStateReader reader, List preferences) throws KeeperException, InterruptedException, IOException {
while (true) {
Stat stat = new Stat();
ZkNodeProps loaded = null;
byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
loaded = ZkNodeProps.load(data);
loaded = loaded.plus("cluster-preferences", preferences);
+ verifyAutoScalingConf(loaded.getProperties());
try {
reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
} catch (KeeperException.BadVersionException bve) {
@@ -557,13 +563,14 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
- private void zkSetClusterPolicy(ZkStateReader reader, List clusterPolicy) throws KeeperException, InterruptedException {
+ private void zkSetClusterPolicy(ZkStateReader reader, List clusterPolicy) throws KeeperException, InterruptedException, IOException {
while (true) {
Stat stat = new Stat();
ZkNodeProps loaded = null;
byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
loaded = ZkNodeProps.load(data);
loaded = loaded.plus("cluster-policy", clusterPolicy);
+ verifyAutoScalingConf(loaded.getProperties());
try {
reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
} catch (KeeperException.BadVersionException bve) {
@@ -574,6 +581,16 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
}
+ private void verifyAutoScalingConf(Map<String, Object> autoScalingConf) throws IOException {
+ try (CloudSolrClient build = new CloudSolrClient.Builder()
+ .withHttpClient(container.getUpdateShardHandler().getHttpClient())
+ .withZkHost(container.getZkController().getZkServerAddress()).build()) {
+ Policy policy = new Policy(autoScalingConf);
+ Policy.Session session = policy.createSession(new SolrClientDataProvider(build));
+ log.debug("Verified autoscaling configuration");
+ }
+ }
+
private Map<String, Object> zkReadAutoScalingConf(ZkStateReader reader) throws KeeperException, InterruptedException {
byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
ZkNodeProps loaded = ZkNodeProps.load(data);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d6b23915/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 9bc7c7a..6f65d93 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -359,8 +360,30 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
" ]" +
"}}";
SolrRequest req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
- NamedList<Object> response = solrClient.request(req);
+ NamedList<Object> response = null;
+ try {
+ response = solrClient.request(req);
+ fail("Adding a policy with 'cores' attribute should not have succeeded.");
+ } catch (HttpSolrClient.RemoteSolrException e) {
+ // expected
+ assertTrue(e.getMessage().contains("cores is only allowed in 'cluster-policy'"));
+ } catch (Exception e) {
+ throw e;
+ }
+
+ setPolicyCommand = "{'set-policy': {" +
+ " 'xyz':[" +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'!overseer', 'replica':0}" +
+ " ]," +
+ " 'policy1':[" +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
+ " ]" +
+ "}}";
+ req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
+ response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
+
byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
ZkNodeProps loaded = ZkNodeProps.load(data);
Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");