You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/05/29 13:33:16 UTC

lucene-solr:feature/autoscaling: SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper

Repository: lucene-solr
Updated Branches:
  refs/heads/feature/autoscaling 011712ff0 -> d6b239158


SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d6b23915
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d6b23915
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d6b23915

Branch: refs/heads/feature/autoscaling
Commit: d6b23915837289ea5c6aa6fde66bf9bee6f36dee
Parents: 011712f
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon May 29 19:03:09 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon May 29 19:03:09 2017 +0530

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  2 ++
 .../cloud/autoscaling/AutoScalingHandler.java   | 33 +++++++++++++++-----
 .../autoscaling/AutoScalingHandlerTest.java     | 25 ++++++++++++++-
 3 files changed, 51 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d6b23915/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5166da7..1ca238d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -142,6 +142,8 @@ Other Changes
 
 * SOLR-10643: Throttling strategy for triggers and policy executions. (shalin)
 
+* SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper. (shalin)
+
 ----------------------
 
 ==================  6.6.0 ==================

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d6b23915/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index cd18307..4d3c566 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -143,7 +143,11 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
           }
         }
       }
-    } finally {
+    } catch (Exception e) {
+      rsp.getValues().add("result", "failure");
+      throw e;
+    }
+    finally {
       RequestHandlerUtils.addExperimentalFormatWarning(rsp);
     }
   }
@@ -182,7 +186,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     }
   }
 
-  private void handleSetClusterPolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+  private void handleSetClusterPolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
     List clusterPolicy = (List) op.getCommandData();
     if (clusterPolicy == null || !(clusterPolicy instanceof List)) {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "A list of cluster policies was not found");
@@ -191,7 +195,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     rsp.getValues().add("result", "success");
   }
 
-  private void handleSetClusterPreferences(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+  private void handleSetClusterPreferences(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
     List preferences = (List) op.getCommandData();
     if (preferences == null || !(preferences instanceof List)) {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "A list of cluster preferences not found");
@@ -200,7 +204,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     rsp.getValues().add("result", "success");
   }
 
-  private void handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+  private void handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
     String policyName = (String) op.getCommandData();
 
     if (policyName.trim().length() == 0) {
@@ -216,7 +220,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     rsp.getValues().add("result", "success");
   }
 
-  private void handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException {
+  private void handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op) throws KeeperException, InterruptedException, IOException {
     Map<String, Object> policies = op.getDataMap();
     for (Map.Entry<String, Object> policy: policies.entrySet()) {
       String policyName = policy.getKey();
@@ -516,7 +520,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     }
   }
 
-  private void zkSetPolicies(ZkStateReader reader, String policyBeRemoved, Map<String, Object> newPolicies) throws KeeperException, InterruptedException {
+  private void zkSetPolicies(ZkStateReader reader, String policyBeRemoved, Map<String, Object> newPolicies) throws KeeperException, InterruptedException, IOException {
     while (true) {
       Stat stat = new Stat();
       ZkNodeProps loaded = null;
@@ -530,6 +534,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
         policies.remove(policyBeRemoved);
       }
       loaded = loaded.plus("policies", policies);
+      verifyAutoScalingConf(loaded.getProperties());
       try {
         reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
       } catch (KeeperException.BadVersionException bve) {
@@ -540,13 +545,14 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     }
   }
 
-  private void zkSetPreferences(ZkStateReader reader, List preferences) throws KeeperException, InterruptedException {
+  private void zkSetPreferences(ZkStateReader reader, List preferences) throws KeeperException, InterruptedException, IOException {
     while (true) {
       Stat stat = new Stat();
       ZkNodeProps loaded = null;
       byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
       loaded = ZkNodeProps.load(data);
       loaded = loaded.plus("cluster-preferences", preferences);
+      verifyAutoScalingConf(loaded.getProperties());
       try {
         reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
       } catch (KeeperException.BadVersionException bve) {
@@ -557,13 +563,14 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     }
   }
 
-  private void zkSetClusterPolicy(ZkStateReader reader, List clusterPolicy) throws KeeperException, InterruptedException {
+  private void zkSetClusterPolicy(ZkStateReader reader, List clusterPolicy) throws KeeperException, InterruptedException, IOException {
     while (true) {
       Stat stat = new Stat();
       ZkNodeProps loaded = null;
       byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
       loaded = ZkNodeProps.load(data);
       loaded = loaded.plus("cluster-policy", clusterPolicy);
+      verifyAutoScalingConf(loaded.getProperties());
       try {
         reader.getZkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(loaded), stat.getVersion(), true);
       } catch (KeeperException.BadVersionException bve) {
@@ -574,6 +581,16 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
     }
   }
 
+  private void verifyAutoScalingConf(Map<String, Object> autoScalingConf) throws IOException {
+    try (CloudSolrClient build = new CloudSolrClient.Builder()
+        .withHttpClient(container.getUpdateShardHandler().getHttpClient())
+        .withZkHost(container.getZkController().getZkServerAddress()).build()) {
+      Policy policy = new Policy(autoScalingConf);
+      Policy.Session session = policy.createSession(new SolrClientDataProvider(build));
+      log.debug("Verified autoscaling configuration");
+    }
+  }
+
   private Map<String, Object> zkReadAutoScalingConf(ZkStateReader reader) throws KeeperException, InterruptedException {
     byte[] data = reader.getZkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
     ZkNodeProps loaded = ZkNodeProps.load(data);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d6b23915/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 9bc7c7a..6f65d93 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -359,8 +360,30 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
         "    ]" +
         "}}";
     SolrRequest req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
-    NamedList<Object> response = solrClient.request(req);
+    NamedList<Object> response = null;
+    try {
+      response = solrClient.request(req);
+      fail("Adding a policy with 'cores' attribute should not have succeeded.");
+    } catch (HttpSolrClient.RemoteSolrException e) {
+      // expected
+      assertTrue(e.getMessage().contains("cores is only allowed in 'cluster-policy'"));
+    } catch (Exception e) {
+      throw e;
+    }
+
+    setPolicyCommand =  "{'set-policy': {" +
+        "    'xyz':[" +
+        "      {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
+        "      {'nodeRole':'!overseer', 'replica':0}" +
+        "    ]," +
+        "    'policy1':[" +
+        "      {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
+        "    ]" +
+        "}}";
+    req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPolicyCommand);
+    response = solrClient.request(req);
     assertEquals(response.get("result").toString(), "success");
+
     byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
     ZkNodeProps loaded = ZkNodeProps.load(data);
     Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");