You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hu...@apache.org on 2020/03/12 20:18:21 UTC
[helix] branch master updated: Add REST APIs for get, set,
update RestConfig (#849)
This is an automated email from the ASF dual-hosted git repository.
hulee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new ecae243 Add REST APIs for get, set, update RestConfig (#849)
ecae243 is described below
commit ecae2431c1cb9f8a4955ac89727ea9fe293d7fdf
Author: Molly Gao <31...@users.noreply.github.com>
AuthorDate: Thu Mar 12 13:18:14 2020 -0700
Add REST APIs for get, set, update RestConfig (#849)
Added endpoints PUT to create, POST to update, and GET to get, DELETE to delete the RestConfig for clusters in ClusterAccessor. Added corresponding methods in ConfigAccessor to be used by ClusterAccessor.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 56 ++++++++++
.../java/org/apache/helix/TestConfigAccessor.java | 88 ++++++++++++++--
.../server/resources/helix/ClusterAccessor.java | 114 +++++++++++++++++++++
.../helix/rest/server/TestClusterAccessor.java | 66 ++++++++++++
4 files changed, 316 insertions(+), 8 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 1596ee3..5d3aded 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -37,6 +37,7 @@ import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.RESTConfig;
import org.apache.helix.model.ResourceConfig;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.helix.util.HelixUtil;
import org.apache.helix.util.StringTemplate;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.zookeeper.api.client.HelixZkClient;
@@ -605,6 +606,61 @@ public class ConfigAccessor {
}
/**
+ * Set RestConfig of a given cluster
+ * @param clusterName the cluster id
+ * @param restConfig the RestConfig to be set to the cluster
+ */
+ public void setRESTConfig(String clusterName, RESTConfig restConfig) {
+ updateRESTConfig(clusterName, restConfig, true);
+ }
+
+ /**
+ * Update RestConfig of a given cluster
+ * @param clusterName the cluster id
+ * @param restConfig the new RestConfig to be set to the cluster
+ */
+ public void updateRESTConfig(String clusterName, RESTConfig restConfig) {
+ updateRESTConfig(clusterName, restConfig, false);
+ }
+
+ private void updateRESTConfig(String clusterName, RESTConfig restConfig, boolean overwrite) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("Fail to update REST config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.REST).forCluster(clusterName).build();
+ String zkPath = scope.getZkPath();
+
+ // Create "/{clusterId}/CONFIGS/REST" if it does not exist
+ String parentPath = HelixUtil.getZkParentPath(zkPath);
+ if (!_zkClient.exists(parentPath)) {
+ ZKUtil.createOrMerge(_zkClient, parentPath, new ZNRecord(parentPath), true, true);
+ }
+
+ if (overwrite) {
+ ZKUtil.createOrReplace(_zkClient, zkPath, restConfig.getRecord(), true);
+ } else {
+ ZKUtil.createOrUpdate(_zkClient, zkPath, restConfig.getRecord(), true, true);
+ }
+ }
+
+ public void deleteRESTConfig(String clusterName) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("Fail to delete REST config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.REST).forCluster(clusterName).build();
+ String zkPath = scope.getZkPath();
+
+ // Check if "/{clusterId}/CONFIGS/REST" exists
+ String parentPath = HelixUtil.getZkParentPath(zkPath);
+ if (!_zkClient.exists(parentPath)) {
+ throw new HelixException("Fail to delete REST config. cluster: " + clusterName + " does not have a rest config."); }
+
+ ZKUtil.dropChildren(_zkClient, parentPath, new ZNRecord(clusterName));
+ }
+
+ /**
* Set ClusterConfig of the given cluster.
* The current Cluster config will be replaced with the given clusterConfig.
* WARNING: This is not thread-safe or concurrent updates safe.
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index 0fa05c3..345cba3 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -24,12 +24,16 @@ import java.util.List;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.model.ConfigScope;
+import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.RESTConfig;
import org.apache.helix.model.builder.ConfigScopeBuilder;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
+
public class TestConfigAccessor extends ZkUnitTestBase {
@Test
public void testBasic() throws Exception {
@@ -113,9 +117,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
"should be [HELIX_ENABLED, HELIX_ENABLED_TIMESTAMP, HELIX_HOST, HELIX_PORT, participantConfigKey]");
Assert.assertEquals(keys.get(4), "participantConfigKey");
- keys =
- configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource",
- "testPartition");
+ keys = configAccessor
+ .getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource", "testPartition");
Assert.assertEquals(keys.size(), 1, "should be [partitionConfigKey]");
Assert.assertEquals(keys.get(0), "partitionConfigKey");
@@ -163,7 +166,6 @@ public class TestConfigAccessor extends ZkUnitTestBase {
configAccessor.close();
configAccessorZkAddr.close();
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
-
}
// HELIX-25: set participant Config should check existence of instance
@@ -183,8 +185,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
try {
configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue");
- Assert
- .fail("Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet");
+ Assert.fail(
+ "Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet");
} catch (HelixException e) {
// OK
}
@@ -193,8 +195,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
try {
configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue");
} catch (Exception e) {
- Assert
- .fail("Except succeed to set participant-config because participant: localhost_12918 has been added to cluster");
+ Assert.fail(
+ "Except succeed to set participant-config because participant: localhost_12918 has been added to cluster");
}
String participantConfigValue = configAccessor.get(participantScope, "participantConfigKey");
@@ -204,4 +206,74 @@ public class TestConfigAccessor extends ZkUnitTestBase {
configAccessor.close();
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
+
+ @Test
+ public void testSetRestConfig() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZKHelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
+ admin.addCluster(clusterName, true);
+ ConfigAccessor configAccessor = new ConfigAccessor(ZK_ADDR);
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.REST).forCluster(clusterName).build();
+ Assert.assertNull(configAccessor.getRESTConfig(clusterName));
+
+ RESTConfig restConfig = new RESTConfig(clusterName);
+ restConfig.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, "TEST_URL");
+ configAccessor.setRESTConfig(clusterName, restConfig);
+ Assert.assertEquals(restConfig, configAccessor.getRESTConfig(clusterName));
+ }
+
+ @Test
+ public void testUpdateAndDeleteRestConfig() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZKHelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
+ admin.addCluster(clusterName, true);
+ ConfigAccessor configAccessor = new ConfigAccessor(ZK_ADDR);
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.REST).forCluster(clusterName).build();
+ Assert.assertNull(configAccessor.getRESTConfig(clusterName));
+
+ // Update
+ // No rest config exist
+ RESTConfig restConfig = new RESTConfig(clusterName);
+ restConfig.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, "TEST_URL");
+ configAccessor.updateRESTConfig(clusterName, restConfig);
+ Assert.assertEquals(restConfig, configAccessor.getRESTConfig(clusterName));
+
+ // Rest config exists
+ restConfig.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, "TEST_URL_2");
+ configAccessor.updateRESTConfig(clusterName, restConfig);
+ Assert.assertEquals(restConfig, configAccessor.getRESTConfig(clusterName));
+
+ // Delete
+ // Existing rest config
+ configAccessor.deleteRESTConfig(clusterName);
+ Assert.assertNull(configAccessor.getRESTConfig(clusterName));
+
+ // Nonexisting rest config
+ admin.addCluster(clusterName, true);
+ try {
+ configAccessor.deleteRESTConfig(clusterName);
+ Assert.fail("Helix exception expected.");
+ } catch (HelixException e) {
+ Assert.assertEquals(e.getMessage(),
+ "Fail to delete REST config. cluster: " + clusterName + " does not have a rest config.");
+ }
+
+ // Nonexisting cluster
+ String anotherClusterName = "anotherCluster";
+ try {
+ configAccessor.deleteRESTConfig(anotherClusterName);
+ Assert.fail("Helix exception expected.");
+ } catch (HelixException e) {
+ Assert.assertEquals(e.getMessage(),
+ "Fail to delete REST config. cluster: " + anotherClusterName + " is NOT setup.");
+ }
+ }
}
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 1fab898..762210a 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -44,6 +44,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey;
import org.apache.helix.PropertyPathBuilder;
+import org.apache.helix.model.RESTConfig;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.zookeeper.api.client.HelixZkClient;
@@ -516,6 +517,119 @@ public class ClusterAccessor extends AbstractHelixResource {
return OK();
}
+ @PUT
+ @Path("{clusterId}/restconfig")
+ public Response createRESTConfig(@PathParam("clusterId") String clusterId,
+ String content) {
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ LOG.error("Failed to deserialize user's input {}. Exception: {}.", content, e);
+ return badRequest("Input is not a valid ZNRecord!");
+ }
+
+ if (!record.getId().equals(clusterId)) {
+ return badRequest("ID does not match the cluster name in input!");
+ }
+
+ RESTConfig config = new RESTConfig(record);
+ ConfigAccessor configAccessor = getConfigAccessor();
+ try {
+ configAccessor.setRESTConfig(clusterId, config);
+ } catch (HelixException ex) {
+ // TODO: Could use a more generic error for HelixException
+ return notFound(ex.getMessage());
+ } catch (Exception ex) {
+ LOG.error("Failed to create rest config, cluster {}, new config: {}. Exception: {}.", clusterId, content, ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
+ @POST
+ @Path("{clusterId}/restconfig")
+ public Response updateRESTConfig(@PathParam("clusterId") String clusterId,
+ @QueryParam("command") String commandStr, String content) {
+ //TODO: abstract out the logic that is duplicated from cluster config methods
+ Command command;
+ try {
+ command = getCommand(commandStr);
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ LOG.error("Failed to deserialize user's input {}. Exception: {}", content, e);
+ return badRequest("Input is not a valid ZNRecord!");
+ }
+
+ RESTConfig config = new RESTConfig(record);
+ ConfigAccessor configAccessor = getConfigAccessor();
+ try {
+ switch (command) {
+ case update:
+ configAccessor.updateRESTConfig(clusterId, config);
+ break;
+ case delete: {
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.REST)
+ .forCluster(clusterId).build();
+ configAccessor.remove(scope, config.getRecord());
+ }
+ break;
+ default:
+ return badRequest("Unsupported command " + commandStr);
+ }
+ } catch (HelixException ex) {
+ return notFound(ex.getMessage());
+ } catch (Exception ex) {
+ LOG.error(
+ "Failed to {} rest config, cluster {}, new config: {}. Exception: {}", command, clusterId, content, ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
+ @GET
+ @Path("{clusterId}/restconfig")
+ public Response getRESTConfig(@PathParam("clusterId") String clusterId) {
+ ConfigAccessor accessor = getConfigAccessor();
+ RESTConfig config = null;
+ try {
+ config = accessor.getRESTConfig(clusterId);
+ } catch (HelixException ex) {
+ LOG.info(
+ "Failed to get rest config for cluster {}, cluster not found. Exception: {}.", clusterId, ex);
+ } catch (Exception ex) {
+ LOG.error("Failed to get rest config for cluster {}. Exception: {}.", clusterId, ex);
+ return serverError(ex);
+ }
+ if (config == null) {
+ return notFound();
+ }
+ return JSONRepresentation(config.getRecord());
+ }
+
+ @DELETE
+ @Path("{clusterId}/restconfig")
+ public Response deleteRESTConfig(@PathParam("clusterId") String clusterId) {
+ ConfigAccessor accessor = getConfigAccessor();
+ try {
+ accessor.deleteRESTConfig(clusterId);
+ } catch (HelixException ex) {
+ LOG.info("Failed to delete rest config for cluster {}, cluster rest config is not found. Exception: {}.", clusterId, ex);
+ return notFound(ex.getMessage());
+ } catch (Exception ex) {
+ LOG.error("Failed to delete rest config, cluster {}, Exception: {}.", clusterId, ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
@GET
@Path("{clusterId}/maintenance")
public Response getClusterMaintenanceMode(@PathParam("clusterId") String clusterId) {
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index e7fda60..b073d22 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -36,6 +36,7 @@ import com.sun.research.ws.wadl.HTTPMethods;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
+import org.apache.helix.model.RESTConfig;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
@@ -576,6 +577,71 @@ public class TestClusterAccessor extends AbstractTestClass {
}
}
+ @Test
+ public void testCreateRESTConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String cluster = _clusters.iterator().next();
+ RESTConfig restConfigRest = new RESTConfig(cluster);
+ restConfigRest.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, "http://*:00");
+ put("clusters/" + cluster + "/restconfig", null, Entity
+ .entity(OBJECT_MAPPER.writeValueAsString(restConfigRest.getRecord()),
+ MediaType.APPLICATION_JSON_TYPE), Response.Status.OK.getStatusCode());
+ RESTConfig restConfigZK = _configAccessor.getRESTConfig(cluster);
+ Assert.assertEquals(restConfigZK, restConfigRest,
+ "rest config from response: " + restConfigRest + " vs rest config actually: "
+ + restConfigZK);
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testCreateRESTConfig")
+ public void testUpdateRESTConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String cluster = _clusters.iterator().next();
+ RESTConfig restConfigRest = new RESTConfig(cluster);
+ // Update an entry
+ restConfigRest.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, "http://*:01");
+ Entity entity = Entity.entity(OBJECT_MAPPER.writeValueAsString(restConfigRest.getRecord()),
+ MediaType.APPLICATION_JSON_TYPE);
+ post("clusters/" + cluster + "/restconfig", ImmutableMap.of("command", Command.update.name()),
+ entity, Response.Status.OK.getStatusCode());
+ RESTConfig restConfigZK = _configAccessor.getRESTConfig(cluster);
+ Assert.assertEquals(restConfigZK, restConfigRest,
+ "rest config from response: " + restConfigRest + " vs rest config actually: "
+ + restConfigZK);
+
+ // Delete an entry
+ restConfigRest.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, null);
+ entity = Entity.entity(OBJECT_MAPPER.writeValueAsString(restConfigRest.getRecord()),
+ MediaType.APPLICATION_JSON_TYPE);
+ post("clusters/" + cluster + "/restconfig", ImmutableMap.of("command", Command.delete.name()),
+ entity, Response.Status.OK.getStatusCode());
+ restConfigZK = _configAccessor.getRESTConfig(cluster);
+ Assert.assertEquals(restConfigZK, new RESTConfig(cluster),
+ "rest config from response: " + new RESTConfig(cluster) + " vs rest config actually: "
+ + restConfigZK);
+
+ // Update a cluster rest config that the cluster does not exist
+ String wrongClusterId = "wrong_cluster_id";
+ restConfigRest = new RESTConfig(wrongClusterId);
+ restConfigRest.set(RESTConfig.SimpleFields.CUSTOMIZED_HEALTH_URL, "http://*:01");
+ entity = Entity.entity(OBJECT_MAPPER.writeValueAsString(restConfigRest.getRecord()),
+ MediaType.APPLICATION_JSON_TYPE);
+ post("clusters/" + wrongClusterId + "/restconfig",
+ ImmutableMap.of("command", Command.update.name()), entity,
+ Response.Status.NOT_FOUND.getStatusCode());
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testUpdateRESTConfig")
+ public void testDeleteRESTConfig() {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String cluster = _clusters.iterator().next();
+ delete("clusters/" + cluster + "/restconfig", Response.Status.OK.getStatusCode());
+ get("clusters/" + cluster + "/restconfig", null, Response.Status.NOT_FOUND.getStatusCode(), true);
+ delete("clusters/" + cluster + "/restconfig", Response.Status.OK.getStatusCode());
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
private ClusterConfig getClusterConfigFromRest(String cluster) throws IOException {
String body = get("clusters/" + cluster + "/configs", null, Response.Status.OK.getStatusCode(), true);