You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2020/04/14 02:20:51 UTC
[helix] branch helix-cloud updated (b9a6c54 -> 1a38abe)
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a change to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git.
from b9a6c54 add Helix cloud interface and implementation skeleton methods
new 19bbf43 Add java API to create cluster with CloudConfig
new 927cc74 Add REST API for Cluster Creation with CloudConfig (#675)
new 0c21ced Add Helix properties factory and class (#653)
new 83afe3a Implement Azure cloud instance information processor (#698)
new cb672d7 Modify participant manager to add cluster auto registration logic (#695)
new 3282760 add one more test for auto registration (#806)
new b9d4627 Change the cluster creation logic (#872)
new 2f6faa9 Add construction of domain in Helix participant logic (#876)
new d4a6293 Change the REST call for delete CloudConfig (#882)
new 1a38abe Add REST and JAVA API to modify existing cloudconfig (#898)
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
helix-core/pom.xml | 5 +
.../main/java/org/apache/helix/ConfigAccessor.java | 44 ++-
.../src/main/java/org/apache/helix/HelixAdmin.java | 15 +-
.../java/org/apache/helix/HelixCloudProperty.java | 183 +++++++++++
.../java/org/apache/helix/HelixManagerFactory.java | 2 +-
.../org/apache/helix/HelixManagerProperty.java | 74 +++++
.../org/apache/helix/HelixPropertyFactory.java | 79 +++++
.../java/org/apache/helix/SystemPropertyKeys.java | 8 +
.../cloud/CloudInstanceInformationProcessor.java | 1 -
.../cloud/azure/AzureCloudInstanceInformation.java | 29 +-
.../AzureCloudInstanceInformationProcessor.java | 118 ++++++-
.../apache/helix/cloud/azure/AzureConstants.java | 6 +
.../helix/manager/zk/ParticipantManager.java | 110 +++++--
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++
.../apache/helix/manager/zk/ZKHelixManager.java | 17 +-
.../java/org/apache/helix/model/CloudConfig.java | 135 +++-----
.../java/org/apache/helix/tools/ClusterSetup.java | 28 +-
.../main/java/org/apache/helix/util/HelixUtil.java | 19 ++
.../dummy.sh => resources/azure-cloud.properties} | 12 +-
...version.properties => helix-manager.properties} | 5 +-
.../java/org/apache/helix/TestConfigAccessor.java | 112 ++++++-
.../org/apache/helix/cloud/MockHttpClient.java | 53 +++
...TestAzureCloudInstanceInformationProcessor.java | 69 ++++
.../manager/MockParticipantManager.java | 10 +-
.../paticipant/TestInstanceAutoJoin.java | 62 +++-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 69 ++++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 11 +
.../apache/helix/model/cloud/TestCloudConfig.java | 60 ++--
.../org/apache/helix/tools/TestClusterSetup.java | 100 +++++-
helix-core/src/test/resources/AzureResponse.json | 104 ++++++
helix-rest/pom.xml | 5 -
.../server/resources/helix/ClusterAccessor.java | 153 ++++++++-
.../helix/rest/server/TestClusterAccessor.java | 364 +++++++++++++++++++++
33 files changed, 1888 insertions(+), 202 deletions(-)
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
copy helix-core/src/main/{scripts/integration-test/testcases/dummy.sh => resources/azure-cloud.properties} (68%)
mode change 100755 => 100644
copy helix-core/src/main/resources/{cluster-manager-version.properties => helix-manager.properties} (88%)
create mode 100644 helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
create mode 100644 helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
create mode 100644 helix-core/src/test/resources/AzureResponse.json
[helix] 09/10: Change the REST call for delete CloudConfig (#882)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit d4a6293987661ece32e9fc874ebf166e7270db43
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 13 13:18:07 2020 -0700
Change the REST call for delete CloudConfig (#882)
Change the delete CloudConfig
Use DELETE instead of POST for deletion of the CloudConfig in REST.
---
.../server/resources/helix/ClusterAccessor.java | 12 ++++++---
.../helix/rest/server/TestClusterAccessor.java | 31 +++++++++++++++-------
2 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index ae31cb5..4e6edaf 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -584,6 +584,14 @@ public class ClusterAccessor extends AbstractHelixResource {
return notFound();
}
+ @DELETE
+ @Path("{clusterId}/cloudconfig")
+ public Response deleteCloudConfig(@PathParam("clusterId") String clusterId) {
+ HelixAdmin admin = getHelixAdmin();
+ admin.removeCloudConfig(clusterId);
+ return OK();
+ }
+
@POST
@Path("{clusterId}/cloudconfig")
public Response updateCloudConfig(@PathParam("clusterId") String clusterId,
@@ -617,10 +625,6 @@ public class ClusterAccessor extends AbstractHelixResource {
}
try {
switch (command) {
- case delete: {
- admin.removeCloudConfig(clusterId);
- }
- break;
case update: {
try {
CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 21a2853..81fc708 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -790,19 +790,32 @@ public class TestClusterAccessor extends AbstractTestClass {
@Test(dependsOnMethods = "testAddCloudConfig")
public void testDeleteCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
- _gSetupTool.addCluster("TestCloud", true);
- String urlBase = "clusters/TestCloud/cloudconfig/";
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
- Map<String, String> map1 = new HashMap<>();
- map1.put("command", AbstractResource.Command.delete.name());
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+ // Read CloudConfig from Zookeeper and make sure it has been created
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNotNull(cloudConfigFromZk);
+ String urlBase = "clusters/" + clusterName + "/cloudconfig/";
- ZNRecord record = new ZNRecord("TestCloud");
- post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
- Response.Status.OK.getStatusCode());
+ delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ _configAccessor = new ConfigAccessor(ZK_ADDR);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk);
System.out.println("End test :" + TestHelper.getTestMethodName());
[helix] 06/10: add one more test for auto registration (#806)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 3282760da9d32522dcb18838c0bb9a24818ec5ba
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Wed Feb 26 10:05:41 2020 -0800
add one more test for auto registration (#806)
Add a test case for participant auto registers to cluster in cloud environment. Note that the final step is expected to fail as the test is not running in a cloud environment.
---
.../paticipant/TestInstanceAutoJoin.java | 62 +++++++++++++++++++---
1 file changed, 54 insertions(+), 8 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
index f0ffc63..5d5ec46 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
@@ -1,16 +1,22 @@
package org.apache.helix.integration.paticipant;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.manager.zk.ZKHelixManager;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.IdealState.RebalanceMode;
import org.apache.helix.model.builder.ConfigScopeBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
+import static org.testng.Assert.*;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -32,14 +38,15 @@ import org.testng.annotations.Test;
public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
String db2 = TEST_DB + "2";
+ String db3 = TEST_DB + "3";
@Test
public void testInstanceAutoJoin() throws Exception {
HelixManager manager = _participants[0];
HelixDataAccessor accessor = manager.getHelixDataAccessor();
- _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "OnlineOffline", RebalanceMode.FULL_AUTO
- + "");
+ _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "OnlineOffline",
+ RebalanceMode.FULL_AUTO + "");
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 1);
String instance2 = "localhost_279699";
@@ -50,8 +57,8 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
Thread.sleep(500);
// Assert.assertFalse(result._thread.isAlive());
- Assert.assertTrue(null == manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2)));
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2)));
ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
@@ -63,15 +70,54 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
Thread.sleep(500);
// Assert.assertTrue(result._thread.isAlive() || result2._thread.isAlive());
for (int i = 0; i < 20; i++) {
- if (null == manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2))) {
+ if (null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2))) {
Thread.sleep(100);
} else
break;
}
- Assert.assertTrue(null != manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2)));
+ Assert.assertTrue(null != manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2)));
newParticipant.syncStop();
}
+
+ @Test(dependsOnMethods = "testInstanceAutoJoin")
+ public void testAutoRegistrationShouldFailWhenWaitingResponse() throws Exception {
+ // Create CloudConfig object and add to config
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
+ CloudConfig cloudConfig = cloudConfigBuilder.build();
+
+ HelixManager manager = _participants[0];
+ HelixDataAccessor accessor = manager.getHelixDataAccessor();
+
+ _gSetupTool.addResourceToCluster(CLUSTER_NAME, db3, 60, "OnlineOffline",
+ RebalanceMode.FULL_AUTO + "");
+ _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db3, 1);
+ String instance3 = "localhost_279700";
+
+ ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
+
+ manager.getConfigAccessor().set(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, "true");
+ // Write the CloudConfig to Zookeeper
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ MockParticipantManager autoParticipant =
+ new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance3);
+ autoParticipant.syncStart();
+
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance3)));
+ try {
+ manager.getConfigAccessor().getInstanceConfig(CLUSTER_NAME, instance3);
+ fail(
+ "Exception should be thrown because the instance cannot be added to the cluster due to the disconnection with Azure endpoint");
+ } catch (HelixException e) {
+
+ }
+ }
}
[helix] 04/10: Implement Azure cloud instance information processor
(#698)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 83afe3a1544b9d32f5813ba67729d9776b93ed2f
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Fri Feb 7 15:24:29 2020 -0800
Implement Azure cloud instance information processor (#698)
Implement Azure cloud instance information processor. The processor performs the functions of fetching and parsing instance information from Azure cloud environment. The endpoint that the processor queries is a globally standard url, called Azure Instance Metadata Service.
---
helix-core/pom.xml | 5 +
.../cloud/CloudInstanceInformationProcessor.java | 1 -
.../cloud/azure/AzureCloudInstanceInformation.java | 11 +-
.../AzureCloudInstanceInformationProcessor.java | 113 +++++++++++++++++++--
.../org/apache/helix/cloud/MockHttpClient.java | 53 ++++++++++
...TestAzureCloudInstanceInformationProcessor.java | 69 +++++++++++++
helix-core/src/test/resources/AzureResponse.json | 104 +++++++++++++++++++
helix-rest/pom.xml | 5 -
8 files changed, 343 insertions(+), 18 deletions(-)
diff --git a/helix-core/pom.xml b/helix-core/pom.xml
index 45b6552..9aecf46 100644
--- a/helix-core/pom.xml
+++ b/helix-core/pom.xml
@@ -159,6 +159,11 @@ under the License.
<artifactId>metrics-core</artifactId>
<version>3.2.3</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.8</version>
+ </dependency>
</dependencies>
<build>
<resources>
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
index a365777..9ff6b00 100644
--- a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
@@ -21,7 +21,6 @@ package org.apache.helix.api.cloud;
import java.util.List;
-
/**
* Generic interface to fetch and parse cloud instance information
*/
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
index f7fd657..1fef205 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -19,17 +19,17 @@ package org.apache.helix.cloud.azure;
* under the License.
*/
+import java.util.HashMap;
import java.util.Map;
import org.apache.helix.api.cloud.CloudInstanceInformation;
-
public class AzureCloudInstanceInformation implements CloudInstanceInformation {
private Map<String, String> _cloudInstanceInfoMap;
/**
* Instantiate the AzureCloudInstanceInformation using each field individually.
- * Users should use AzureCloudInstanceInformation.Builder to create information.
+ * Users should use AzureCloudInstanceInformation.Builder to set field information.
* @param cloudInstanceInfoMap
*/
protected AzureCloudInstanceInformation(Map<String, String> cloudInstanceInfoMap) {
@@ -42,10 +42,11 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
}
public static class Builder {
- private Map<String, String> _cloudInstanceInfoMap = null;
+
+ private final Map<String, String> _cloudInstanceInfoMap = new HashMap<>();
public AzureCloudInstanceInformation build() {
- return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
+ return new AzureCloudInstanceInformation(new HashMap<>(_cloudInstanceInfoMap));
}
public Builder setInstanceName(String name) {
@@ -68,4 +69,4 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
return this;
}
}
-}
\ No newline at end of file
+}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 84a102c..85e2bb5 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -19,38 +19,137 @@ package org.apache.helix.cloud.azure;
* under the License.
*/
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
+import javax.net.ssl.SSLException;
+import org.apache.helix.HelixCloudProperty;
+import org.apache.helix.HelixException;
import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.EntityUtils;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+public class AzureCloudInstanceInformationProcessor
+ implements CloudInstanceInformationProcessor<String> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AzureCloudInstanceInformationProcessor.class);
+ private final CloseableHttpClient _closeableHttpClient;
+ private final HelixCloudProperty _helixCloudProperty;
+ private final String COMPUTE = "compute";
+ private final String INSTANCE_NAME = "vmId";
+ private final String DOMAIN = "platformFaultDomain";
+ private final String INSTANCE_SET_NAME = "vmScaleSetName";
-public class AzureCloudInstanceInformationProcessor implements CloudInstanceInformationProcessor<String> {
+ public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty) {
+ _helixCloudProperty = helixCloudProperty;
- public AzureCloudInstanceInformationProcessor() {
+ RequestConfig requestConifg = RequestConfig.custom()
+ .setConnectionRequestTimeout((int) helixCloudProperty.getCloudRequestTimeout())
+ .setConnectTimeout((int) helixCloudProperty.getCloudConnectionTimeout()).build();
+
+ HttpRequestRetryHandler httpRequestRetryHandler =
+ (IOException exception, int executionCount, HttpContext context) -> {
+ LOG.warn("Execution count: " + executionCount + ".", exception);
+ return !(executionCount >= helixCloudProperty.getCloudMaxRetry()
+ || exception instanceof InterruptedIOException
+ || exception instanceof UnknownHostException || exception instanceof SSLException);
+ };
+
+ //TODO: we should regularize the way how httpClient should be used throughout Helix. e.g. Helix-rest could also use in the same way
+ _closeableHttpClient = HttpClients.custom().setDefaultRequestConfig(requestConifg)
+ .setRetryHandler(httpRequestRetryHandler).build();
}
/**
- * fetch the raw Azure cloud instance information
+ * This constructor is for unit test purpose only.
+ * User could provide helixCloudProperty and a mocked http client to test the functionality of
+ * this class.
+ */
+ public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty,
+ CloseableHttpClient closeableHttpClient) {
+ _helixCloudProperty = helixCloudProperty;
+ _closeableHttpClient = closeableHttpClient;
+ }
+
+ /**
+ * Fetch raw Azure cloud instance information based on the urls provided
* @return raw Azure cloud instance information
*/
@Override
public List<String> fetchCloudInstanceInformation() {
List<String> response = new ArrayList<>();
- //TODO: implement the fetching logic
+ for (String url : _helixCloudProperty.getCloudInfoSources()) {
+ response.add(getAzureCloudInformationFromUrl(url));
+ }
return response;
}
/**
+ * Query Azure Instance Metadata Service to get the instance(VM) information
+ * @return raw Azure cloud instance information
+ */
+ private String getAzureCloudInformationFromUrl(String url) {
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setHeader("Metadata", "true");
+
+ try {
+ CloseableHttpResponse response = _closeableHttpClient.execute(httpGet);
+ if (response == null || response.getStatusLine().getStatusCode() != 200) {
+ String errorMsg = String.format(
+ "Failed to get an HTTP Response for the request. Response: {}. Status code: {}",
+ (response == null ? "NULL" : response.getStatusLine().getReasonPhrase()),
+ response.getStatusLine().getStatusCode());
+ throw new HelixException(errorMsg);
+ }
+ String responseString = EntityUtils.toString(response.getEntity());
+ LOG.info("VM instance information query result: {}", responseString);
+ return responseString;
+ } catch (IOException e) {
+ throw new HelixException(
+ String.format("Failed to get Azure cloud instance information from url {}", url), e);
+ }
+ }
+
+ /**
* Parse raw Azure cloud instance information.
* @return required azure cloud instance information
*/
@Override
public AzureCloudInstanceInformation parseCloudInstanceInformation(List<String> responses) {
AzureCloudInstanceInformation azureCloudInstanceInformation = null;
- //TODO: implement the parsing logic
+ if (responses.size() > 1) {
+ throw new HelixException("Multiple responses are not supported for Azure now");
+ }
+ String response = responses.get(0);
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ JsonNode jsonNode = mapper.readTree(response);
+ JsonNode computeNode = jsonNode.path(COMPUTE);
+ if (!computeNode.isMissingNode()) {
+ String vmName = computeNode.path(INSTANCE_NAME).getTextValue();
+ String platformFaultDomain = computeNode.path(DOMAIN).getTextValue();
+ String vmssName = computeNode.path(INSTANCE_SET_NAME).getValueAsText();
+ AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
+ builder.setInstanceName(vmName).setFaultDomain(platformFaultDomain)
+ .setInstanceSetName(vmssName);
+ azureCloudInstanceInformation = builder.build();
+ }
+ } catch (IOException e) {
+ throw new HelixException(String.format("Error in parsing cloud instance information: {}", response, e));
+ }
return azureCloudInstanceInformation;
}
}
-
-
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java b/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
new file mode 100644
index 0000000..03e687e
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
@@ -0,0 +1,53 @@
+package org.apache.helix.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.InputStream;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+
+/**
+ * Mock a http client and provide response using resource file. This is for unit test purpose only.
+ */
+public class MockHttpClient {
+ protected CloseableHttpClient createMockHttpClient(String file) throws Exception {
+ InputStream responseInputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(file);
+ HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+ StatusLine statusLine = Mockito.mock(StatusLine.class);
+
+ CloseableHttpResponse mockCloseableHttpResponse = Mockito.mock(CloseableHttpResponse.class);
+ CloseableHttpClient mockCloseableHttpClient = Mockito.mock(CloseableHttpClient.class);
+
+ Mockito.when(httpEntity.getContent()).thenReturn(responseInputStream);
+ Mockito.when(mockCloseableHttpClient.execute(Matchers.any(HttpGet.class))).thenReturn(mockCloseableHttpResponse);
+ Mockito.when(mockCloseableHttpResponse.getEntity()).thenReturn(httpEntity);
+ Mockito.when(mockCloseableHttpResponse.getStatusLine()).thenReturn(statusLine);
+ Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+
+ return mockCloseableHttpClient;
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..10ba42d
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -0,0 +1,69 @@
+package org.apache.helix.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+import org.apache.helix.HelixCloudProperty;
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+import org.apache.helix.cloud.azure.AzureCloudInstanceInformation;
+import org.apache.helix.cloud.azure.AzureCloudInstanceInformationProcessor;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.CloudConfig;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+/**
+ * Unit test for {@link AzureCloudInstanceInformationProcessor}
+ */
+public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
+
+ @Test()
+ public void testAzureCloudInstanceInformationProcessing() throws Exception {
+ String responseFile = "AzureResponse.json";
+
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
+ HelixCloudProperty helixCloudProperty = new HelixCloudProperty(cloudConfigBuilder.build());
+ AzureCloudInstanceInformationProcessor processor = new AzureCloudInstanceInformationProcessor(
+ helixCloudProperty, createMockHttpClient(responseFile));
+ List<String> response = processor.fetchCloudInstanceInformation();
+
+ Assert.assertEquals(response.size(), 1);
+ Assert.assertNotNull(response.get(0));
+
+ // Verify the response from mock http client
+ AzureCloudInstanceInformation azureCloudInstanceInformation =
+ processor.parseCloudInstanceInformation(response);
+ Assert.assertEquals(azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()), "2");
+ Assert.assertEquals(
+ azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()),
+ "test-helix");
+ Assert.assertEquals(
+ azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
+ "d2b921cc-c16c-41f7-a86d-a445eac6ec26");
+ }
+}
diff --git a/helix-core/src/test/resources/AzureResponse.json b/helix-core/src/test/resources/AzureResponse.json
new file mode 100644
index 0000000..dfe13ad
--- /dev/null
+++ b/helix-core/src/test/resources/AzureResponse.json
@@ -0,0 +1,104 @@
+{
+ "compute": {
+ "azEnvironment": "AzurePublicCloud",
+ "customData": "",
+ "location": "southcentralus",
+ "name": "test-helix_1",
+ "offer": "",
+ "osType": "Linux",
+ "placementGroupId": "81e605b2-a807-48ee-a84a-63c76a9c9543",
+ "plan": {
+ "name": "",
+ "product": "",
+ "publisher": ""
+ },
+ "platformFaultDomain": "2",
+ "platformUpdateDomain": "2",
+ "provider": "Microsoft.Compute",
+ "publicKeys": [],
+ "publisher": "",
+ "resourceGroupName": "scus-lpsazureei1-app-rg",
+ "resourceId": "/subscriptions/c9a251d8-1272-4c0f-8055-8271bbc1d677/resourceGroups/scus-lpsazureei1-app-rg/providers/Microsoft.Compute/virtualMachines/test-helix_2",
+ "sku": "",
+ "storageProfile": {
+ "dataDisks": [],
+ "imageReference": {
+ "id": "/subscriptions/7dd5a659-67c4-441c-ac0b-d48b7a029668/resourceGroups/scus-infra-app-rg/providers/Microsoft.Compute/galleries/pieimagerepo/images/FastCOP4/versions/190924.1.1",
+ "offer": "",
+ "publisher": "",
+ "sku": "",
+ "version": ""
+ },
+ "osDisk": {
+ "caching": "ReadWrite",
+ "createOption": "FromImage",
+ "diskSizeGB": "32",
+ "encryptionSettings": {
+ "enabled": "false"
+ },
+ "image": {
+ "uri": ""
+ },
+ "managedDisk": {
+ "id": "/subscriptions/c9a251d8-1272-4c0f-8055-8271bbc1d677/resourceGroups/scus-lpsazureei1-app-rg/providers/Microsoft.Compute/disks/test-helix_test-helix_2_OsDisk_1_124c3534b8e848e296ec22b24d44c027",
+ "storageAccountType": "Standard_LRS"
+ },
+ "name": "test-helix_test-helix_2_OsDisk_1_124c3534b8e848e296ec22b24d44c027",
+ "osType": "Linux",
+ "vhd": {
+ "uri": ""
+ },
+ "writeAcceleratorEnabled": "false"
+ }
+ },
+ "subscriptionId": "c9a251d8-1272-4c0f-8055-8271bbc1d677",
+ "tags": "automation:terraform;environment:dev;module:lid-vmss;moduleVersion:0.0.1",
+ "tagsList": [
+ {
+ "name": "automation",
+ "value": "terraform"
+ },
+ {
+ "name": "environment",
+ "value": "dev"
+ },
+ {
+ "name": "module",
+ "value": "lid-vmss"
+ },
+ {
+ "name": "moduleVersion",
+ "value": "0.0.1"
+ }
+ ],
+ "version": "",
+ "vmId": "d2b921cc-c16c-41f7-a86d-a445eac6ec26",
+ "vmScaleSetName": "test-helix",
+ "vmSize": "Standard_D16s_v3",
+ "zone": ""
+ },
+ "network": {
+ "interface": [
+ {
+ "ipv4": {
+ "ipAddress": [
+ {
+ "privateIpAddress": "10.3.64.12",
+ "publicIpAddress": ""
+ }
+ ],
+ "subnet": [
+ {
+ "address": "10.3.64.0",
+ "prefix": "19"
+ }
+ ]
+ },
+ "ipv6": {
+ "ipAddress": []
+ },
+ "macAddress": "000D3A769010"
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml
index 236669d..34293e6 100644
--- a/helix-rest/pom.xml
+++ b/helix-rest/pom.xml
@@ -67,11 +67,6 @@ under the License.
<version>3.8.1</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.8</version>
- </dependency>
- <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.1.0.RC0</version>
[helix] 03/10: Add Helix properties factory and class (#653)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 0c21ced9a9d4f18212e207148fbdcb15f523e15d
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Jan 21 14:05:50 2020 -0800
Add Helix properties factory and class (#653)
- Add Helix property singleton factory. The factory returns Helix property with default values, and clients may override these values;
- Add Helix manager property. It further holds Helix cloud properties and some other properties specific to the manager, defined in helix-manager.properties file.
- Add Helix cloud property. It holds cloud related properties, which comes from cloud config and client specified file, e.g., for Azure, the file is azure-cloud.properties.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 2 -
.../java/org/apache/helix/HelixCloudProperty.java | 183 +++++++++++++++++++++
.../java/org/apache/helix/HelixManagerFactory.java | 2 +-
.../org/apache/helix/HelixManagerProperty.java | 74 +++++++++
.../org/apache/helix/HelixPropertyFactory.java | 79 +++++++++
.../java/org/apache/helix/SystemPropertyKeys.java | 8 +
.../cloud/azure/AzureCloudInstanceInformation.java | 18 +-
.../helix/manager/zk/ParticipantManager.java | 10 ++
.../apache/helix/manager/zk/ZKHelixManager.java | 17 +-
.../java/org/apache/helix/model/CloudConfig.java | 36 +++-
.../src/main/resources/azure-cloud.properties | 25 +++
.../src/main/resources/helix-manager.properties | 24 +++
12 files changed, 453 insertions(+), 25 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 97bfb34..e1d6399 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -609,9 +609,7 @@ public class ConfigAccessor {
/**
* Get RestConfig of the given cluster.
- *
* @param clusterName The cluster
- *
* @return The instance of {@link RESTConfig}
*/
public RESTConfig getRESTConfig(String clusterName) {
diff --git a/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
new file mode 100644
index 0000000..554c595
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
@@ -0,0 +1,183 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hold helix cloud properties read from CloudConfig and user defined files. Clients may override
+ * the fields from their application.
+ */
+public class HelixCloudProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixCloudProperty.class.getName());
+ private static final String AZURE_CLOUD_PROPERTY_FILE = SystemPropertyKeys.AZURE_CLOUD_PROPERTIES;
+ private static final String CLOUD_INFO_SOURCE = "cloud_info_source";
+ private static final String CLOUD_INFO_PROCESSFOR_NAME = "cloud_info_processor_name";
+ private static final String CLOUD_MAX_RETRY = "cloud_max_retry";
+ private static final String CONNECTION_TIMEOUT_MS = "connection_timeout_ms";
+ private static final String REQUEST_TIMEOUT_MS = "request_timeout_ms";
+
+ // Denote whether the instance is considered as in a cloud environment.
+ private boolean _isCloudEnabled;
+
+ // Unique id of the cloud environment where the instance is in.
+ private String _cloudId;
+
+ // Cloud environment provider, e.g. Azure, AWS, GCP, etc.
+ private String _cloudProvider;
+
+ // The sources where the cloud instance information can be retrieved from.
+ private List<String> _cloudInfoSources;
+
+ // The name of the function that will fetch and parse cloud instance information.
+ private String _cloudInfoProcessorName;
+
+ // Http max retry times when querying the cloud instance information from cloud environment.
+ private int _cloudMaxRetry;
+
+ // Http connection time when querying the cloud instance information from cloud environment.
+ private long _cloudConnectionTimeout;
+
+ // Http request timeout when querying the cloud instance information from cloud environment.
+ private long _cloudRequestTimeout;
+
+ // Other customized properties that may be used.
+ private Properties _customizedCloudProperties = new Properties();
+
+ /**
+ * Initialize Helix Cloud Property based on the provider
+ * @param
+ */
+ public HelixCloudProperty(CloudConfig cloudConfig) {
+ setCloudEndabled(cloudConfig.isCloudEnabled());
+ if (cloudConfig.isCloudEnabled()) {
+ setCloudId(cloudConfig.getCloudID());
+ setCloudProvider(cloudConfig.getCloudProvider());
+ switch (CloudProvider.valueOf(cloudConfig.getCloudProvider())) {
+ case AZURE:
+ Properties azureProperties = new Properties();
+ try {
+ InputStream stream = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(AZURE_CLOUD_PROPERTY_FILE);
+ azureProperties.load(stream);
+ } catch (IOException e) {
+ String errMsg =
+ "failed to open Helix Azure cloud properties file: " + AZURE_CLOUD_PROPERTY_FILE;
+ throw new IllegalArgumentException(errMsg, e);
+ }
+ LOG.info("Successfully loaded Helix Azure cloud properties: {}", azureProperties);
+ setCloudInfoSources(
+ Collections.singletonList(azureProperties.getProperty(CLOUD_INFO_SOURCE)));
+ setCloudInfoProcessorName(azureProperties.getProperty(CLOUD_INFO_PROCESSFOR_NAME));
+ setCloudMaxRetry(Integer.valueOf(azureProperties.getProperty(CLOUD_MAX_RETRY)));
+ setCloudConnectionTimeout(Long.valueOf(azureProperties.getProperty(CONNECTION_TIMEOUT_MS)));
+ setCloudRequestTimeout(Long.valueOf(azureProperties.getProperty(REQUEST_TIMEOUT_MS)));
+ break;
+ case CUSTOMIZED:
+ setCloudInfoSources(cloudConfig.getCloudInfoSources());
+ setCloudInfoProcessorName(cloudConfig.getCloudInfoProcessorName());
+ break;
+ default:
+ throw new HelixException(
+ String.format("Unsupported cloud provider: %s", cloudConfig.getCloudProvider()));
+ }
+ }
+ }
+
+ public boolean getCloudEnabled() {
+ return _isCloudEnabled;
+ }
+
+ public String getCloudId() {
+ return _cloudId;
+ }
+
+ public String getCloudProvider() {
+ return _cloudProvider;
+ }
+
+ public List<String> getCloudInfoSources() {
+ return _cloudInfoSources;
+ }
+
+ public String getCloudInfoProcessorName() {
+ return _cloudInfoProcessorName;
+ }
+
+ public int getCloudMaxRetry() {
+ return _cloudMaxRetry;
+ }
+
+ public long getCloudConnectionTimeout() {
+ return _cloudConnectionTimeout;
+ }
+
+ public long getCloudRequestTimeout() {
+ return _cloudRequestTimeout;
+ }
+
+ public Properties getCustomizedCloudProperties() {
+ return _customizedCloudProperties;
+ }
+
+ public void setCloudEndabled(boolean isCloudEnabled) {
+ _isCloudEnabled = isCloudEnabled;
+ }
+
+ public void setCloudId(String cloudId) {
+ _cloudId = cloudId;
+ }
+
+ public void setCloudProvider(String cloudProvider) {
+ _cloudProvider = cloudProvider;
+ }
+
+ public void setCloudInfoSources(List<String> sources) {
+ _cloudInfoSources = sources;
+ }
+
+ public void setCloudInfoProcessorName(String cloudInfoProcessorName) {
+ _cloudInfoProcessorName = cloudInfoProcessorName;
+ }
+
+ public void setCloudMaxRetry(int cloudMaxRetry) {
+ _cloudMaxRetry = cloudMaxRetry;
+ }
+
+ public void setCloudConnectionTimeout(long cloudConnectionTimeout) {
+ _cloudConnectionTimeout = cloudConnectionTimeout;
+ }
+
+ public void setCloudRequestTimeout(long cloudRequestTimeout) {
+ _cloudRequestTimeout = cloudRequestTimeout;
+ }
+
+ public void setCustomizedCloudProperties(Properties customizedCloudProperties) {
+ _customizedCloudProperties.putAll(customizedCloudProperties);
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java b/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
index 7d8038d..ee53c1d 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
* Obtain one of a set of Helix cluster managers, organized by the backing system.
*/
public final class HelixManagerFactory {
- private static final Logger logger = LoggerFactory.getLogger(HelixManagerFactory.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HelixManagerFactory.class);
/**
* Construct a zk-based cluster manager that enforces all types (PARTICIPANT, CONTROLLER, and
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java b/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
new file mode 100644
index 0000000..2ec26a7
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
@@ -0,0 +1,74 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Properties;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hold Helix manager properties. The manager properties further hold Helix cloud properties
+ * and some other properties specific for the manager.
+ */
+public class HelixManagerProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixManagerProperty.class.getName());
+ private String _version;
+ private long _healthReportLatency;
+ private HelixCloudProperty _helixCloudProperty;
+
+ /**
+ * Initialize Helix manager property with default value
+ * @param helixManagerProperties helix manager related properties input as a map
+ * @param cloudConfig cloudConfig read from Zookeeper
+ */
+ public HelixManagerProperty(Properties helixManagerProperties, CloudConfig cloudConfig) {
+ _helixCloudProperty = new HelixCloudProperty(cloudConfig);
+ setVersion(helixManagerProperties.getProperty(SystemPropertyKeys.HELIX_MANAGER_VERSION));
+ setHealthReportLatency(
+ helixManagerProperties.getProperty(SystemPropertyKeys.PARTICIPANT_HEALTH_REPORT_LATENCY));
+ }
+
+ public HelixCloudProperty getHelixCloudProperty() {
+ return _helixCloudProperty;
+ }
+
+ public String getVersion() {
+ return _version;
+ }
+
+ public long getHealthReportLatency() {
+ return _healthReportLatency;
+ }
+
+ public void setHelixCloudProperty(HelixCloudProperty helixCloudProperty) {
+ _helixCloudProperty = helixCloudProperty;
+ }
+
+ public void setVersion(String version) {
+ _version = version;
+ }
+
+ public void setHealthReportLatency(String latency) {
+ _healthReportLatency = Long.valueOf(latency);
+ }
+
+ // TODO: migrate all other participant related properties to this file.
+}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
new file mode 100644
index 0000000..fa394a2
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
@@ -0,0 +1,79 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Singleton factory that builds different types of Helix property, e.g. Helix manager property.
+ */
+public final class HelixPropertyFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixPropertyFactory.class);
+ private static final String HELIX_PARTICIPANT_PROPERTY_FILE =
+ SystemPropertyKeys.HELIX_MANAGER_PROPERTIES;
+
+ private static class SingletonHelper {
+ private static final HelixPropertyFactory INSTANCE = new HelixPropertyFactory();
+ }
+
+ public static HelixPropertyFactory getInstance() {
+ return SingletonHelper.INSTANCE;
+ }
+
+ /**
+ * Retrieve Helix manager property. It returns the property object with default values.
+ * Clients may override these values.
+ */
+ public HelixManagerProperty getHelixManagerProperty(String zkAddress, String clusterName) {
+ ConfigAccessor configAccessor = new ConfigAccessor(zkAddress);
+ CloudConfig cloudConfig;
+ // The try-catch logic is for backward compatibility reason only. Even if the cluster is not set
+ // up yet, constructing a new ZKHelixManager should not throw an exception
+ try {
+ cloudConfig =
+ configAccessor.getCloudConfig(clusterName) == null ? buildEmptyCloudConfig(clusterName)
+ : configAccessor.getCloudConfig(clusterName);
+ } catch (HelixException e) {
+ cloudConfig = buildEmptyCloudConfig(clusterName);
+ }
+ Properties properties = new Properties();
+ try {
+ InputStream stream = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(HELIX_PARTICIPANT_PROPERTY_FILE);
+ properties.load(stream);
+ } catch (IOException e) {
+ String errMsg = String.format("failed to open Helix participant properties file: %s",
+ HELIX_PARTICIPANT_PROPERTY_FILE);
+ throw new IllegalArgumentException(errMsg, e);
+ }
+ LOG.info("HelixPropertyFactory successfully loaded helix participant properties: {}",
+ properties);
+ return new HelixManagerProperty(properties, cloudConfig);
+ }
+
+ public static CloudConfig buildEmptyCloudConfig(String clusterName) {
+ return new CloudConfig.Builder().setCloudEnabled(false).build();
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java b/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
index 1a6a797..b1735a0 100644
--- a/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
+++ b/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
@@ -1,6 +1,14 @@
package org.apache.helix;
public class SystemPropertyKeys {
+ // Used to compose default values in HelixManagerProperty
+ public static final String HELIX_MANAGER_PROPERTIES = "helix-manager.properties";
+
+ public static final String HELIX_MANAGER_VERSION = "clustermanager.version";
+
+ // Used to compose default values in HelixCloudProperty when cloud provider is Azure
+ public static final String AZURE_CLOUD_PROPERTIES = "azure-cloud.properties";
+
// Task Driver
public static final String TASK_CONFIG_LIMITATION = "helixTask.configsLimitation";
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
index 511f3e3..f7fd657 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -48,24 +48,18 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
}
- /**
- * Default constructor
- */
- public Builder() {
- }
-
- public Builder setInstanceName(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), v);
+ public Builder setInstanceName(String name) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), name);
return this;
}
- public Builder setFaultDomain(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), v);
+ public Builder setFaultDomain(String faultDomain) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), faultDomain);
return this;
}
- public Builder setInstanceSetName(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), v);
+ public Builder setInstanceSetName(String instanceSetName) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), instanceSetName);
return this;
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 76cb791..9c5d602 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -33,6 +33,7 @@ import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
import org.apache.helix.PreConnectCallback;
@@ -76,9 +77,17 @@ public class ParticipantManager {
final StateMachineEngine _stateMachineEngine;
final LiveInstanceInfoProvider _liveInstanceInfoProvider;
final List<PreConnectCallback> _preConnectCallbacks;
+ final HelixManagerProperty _helixManagerProperty;
+ @Deprecated
public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout,
LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks) {
+ this(manager, zkclient, sessionTimeout, liveInstanceInfoProvider, preConnectCallbacks, null);
+ }
+
+ public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout,
+ LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks,
+ HelixManagerProperty helixManagerProperty) {
_zkclient = zkclient;
_manager = manager;
_clusterName = manager.getClusterName();
@@ -94,6 +103,7 @@ public class ParticipantManager {
_stateMachineEngine = manager.getStateMachineEngine();
_liveInstanceInfoProvider = liveInstanceInfoProvider;
_preConnectCallbacks = preConnectCallbacks;
+ _helixManagerProperty = helixManagerProperty;
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index df2dccc..693dc17 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -41,6 +41,8 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerProperties;
+import org.apache.helix.HelixManagerProperty;
+import org.apache.helix.HelixPropertyFactory;
import org.apache.helix.HelixTimerTask;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
@@ -106,6 +108,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
private final List<PreConnectCallback> _preConnectCallbacks;
protected final List<CallbackHandler> _handlers;
private final HelixManagerProperties _properties;
+ private final HelixManagerProperty _helixManagerProperty;
private final HelixManagerStateListener _stateListener;
/**
@@ -202,9 +205,16 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
public ZKHelixManager(String clusterName, String instanceName, InstanceType instanceType,
String zkAddress, HelixManagerStateListener stateListener) {
+ this(clusterName, instanceName, instanceType, zkAddress, stateListener,
+ HelixPropertyFactory.getInstance().getHelixManagerProperty(zkAddress, clusterName));
+ }
+
+ public ZKHelixManager(String clusterName, String instanceName, InstanceType instanceType,
+ String zkAddress, HelixManagerStateListener stateListener,
+ HelixManagerProperty helixManagerProperty) {
- LOG.info(
- "Create a zk-based cluster manager. zkSvr: " + zkAddress + ", clusterName: " + clusterName + ", instanceName: " + instanceName + ", type: " + instanceType);
+ LOG.info("Create a zk-based cluster manager. zkSvr: " + zkAddress + ", clusterName: "
+ + clusterName + ", instanceName: " + instanceName + ", type: " + instanceType);
_zkAddress = zkAddress;
_clusterName = clusterName;
@@ -244,6 +254,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
_stateListener = stateListener;
+ _helixManagerProperty = helixManagerProperty;
/**
* use system property if available
@@ -1203,7 +1214,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
_participantManager =
new ParticipantManager(this, _zkclient, _sessionTimeout, _liveInstanceInfoProvider,
- _preConnectCallbacks);
+ _preConnectCallbacks, _helixManagerProperty);
_participantManager.handleNewSession();
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index f6c279c..79c7330 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -71,6 +71,26 @@ public class CloudConfig extends HelixProperty {
_record.setMapFields(record.getMapFields());
}
+ /**
+ * Instantiate the config using each field individually.
+ * Users should use CloudConfig.Builder to create CloudConfig.
+ * @param cluster
+ * @param enabled
+ * @param cloudID
+ */
+ public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
+ List<String> cloudInfoSource, String cloudProcessorName) {
+ super(cluster);
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ if (enabled == true) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
+ }
+ }
+ }
/**
* Enable/Disable the CLOUD_ENABLED field.
@@ -226,14 +246,16 @@ public class CloudConfig extends HelixProperty {
}
private void validate() {
- if (this.getCloudProvider() == null) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
- } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
- if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
- || this.getCloudInfoSources().size() == 0) {
+ if (this.getCloudEnabled()) {
+ if (this.getCloudProvider() == null) {
throw new HelixException(
- "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
+ if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
+ || this.getCloudInfoSources().size() == 0) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ }
}
}
}
diff --git a/helix-core/src/main/resources/azure-cloud.properties b/helix-core/src/main/resources/azure-cloud.properties
new file mode 100644
index 0000000..b5321fc
--- /dev/null
+++ b/helix-core/src/main/resources/azure-cloud.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This is the globally fixed value for Azure Metadata Instance Service endpoint
+cloud_info_source=http://169.254.169.254/metadata/instance?api-version=2019-06-04
+cloud_info_processor_name=AzureCloudInstanceInformationProcessor
+cloud_max_retry=5
+connection_timeout_ms=5000
+request_timeout_ms=5000
\ No newline at end of file
diff --git a/helix-core/src/main/resources/helix-manager.properties b/helix-core/src/main/resources/helix-manager.properties
new file mode 100644
index 0000000..eafcb3a
--- /dev/null
+++ b/helix-core/src/main/resources/helix-manager.properties
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+clustermanager.version=${project.version}
+
+minimum_supported_version.participant=0.4
+
+helixmanager.participantHealthReport.reportLatency=60000
\ No newline at end of file
[helix] 10/10: Add REST and JAVA API to modify existing cloudconfig
(#898)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 1a38abe20dbb326a5c09e901a8131ee8f15bddf8
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Mar 17 13:07:46 2020 -0700
Add REST and JAVA API to modify existing cloudconfig (#898)
The new APIs are added to update/delete cloudconfig entries.
Tests have been modified accordingly.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 42 +++++++++++++++-
.../AzureCloudInstanceInformationProcessor.java | 8 ++--
.../java/org/apache/helix/model/CloudConfig.java | 2 +-
.../java/org/apache/helix/TestConfigAccessor.java | 56 ++++++++++++++++++++--
...TestAzureCloudInstanceInformationProcessor.java | 8 ++--
.../server/resources/helix/ClusterAccessor.java | 13 +++--
.../helix/rest/server/TestClusterAccessor.java | 50 ++++++++++++++++++-
7 files changed, 157 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index e1d6399..98e45e1 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -604,7 +604,47 @@ public class ConfigAccessor {
return null;
}
- return new CloudConfig.Builder(record).build();
+ return new CloudConfig(record);
+ }
+
+ /**
+ * Delete cloud config fields (not the whole config)
+ * @param clusterName
+ * @param cloudConfig
+ */
+ public void deleteCloudConfigFields(String clusterName, CloudConfig cloudConfig) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("fail to delete cloud config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ remove(scope, cloudConfig.getRecord());
+ }
+
+ /**
+ * Update cloud config
+ * @param clusterName
+ * @param cloudConfig
+ */
+ public void updateCloudConfig(String clusterName, CloudConfig cloudConfig) {
+ updateCloudConfig(clusterName, cloudConfig, false);
+ }
+
+ private void updateCloudConfig(String clusterName, CloudConfig cloudConfig, boolean overwrite) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("Fail to update cloud config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ String zkPath = scope.getZkPath();
+
+ if (overwrite) {
+ ZKUtil.createOrReplace(_zkClient, zkPath, cloudConfig.getRecord(), true);
+ } else {
+ ZKUtil.createOrUpdate(_zkClient, zkPath, cloudConfig.getRecord(), true, true);
+ }
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 464ff55..f0f75f7 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -145,15 +145,15 @@ public class AzureCloudInstanceInformationProcessor
String azureTopology = AzureConstants.AZURE_TOPOLOGY;
String[] parts = azureTopology.trim().split("/");
//The hostname will be filled in by each participant
- String domain = parts[0] + "=" + platformFaultDomain + "," + parts[1] + "=";
+ String domain = parts[1] + "=" + platformFaultDomain + "," + parts[2] + "=";
AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
- builder.setInstanceName(vmName).setFaultDomain(domain)
- .setInstanceSetName(vmssName);
+ builder.setInstanceName(vmName).setFaultDomain(domain).setInstanceSetName(vmssName);
azureCloudInstanceInformation = builder.build();
}
} catch (IOException e) {
- throw new HelixException(String.format("Error in parsing cloud instance information: {}", response, e));
+ throw new HelixException(
+ String.format("Error in parsing cloud instance information: {}", response, e));
}
return azureCloudInstanceInformation;
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index 79c7330..89fcd43 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -64,7 +64,7 @@ public class CloudConfig extends HelixProperty {
* The constructor from the ZNRecord.
* @param record
*/
- private CloudConfig(ZNRecord record) {
+ public CloudConfig(ZNRecord record) {
super(CLOUD_CONFIG_KW);
_record.setSimpleFields(record.getSimpleFields());
_record.setListFields(record.getListFields());
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index e62bf81..63937fa 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -227,7 +227,7 @@ public class TestConfigAccessor extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -237,10 +237,10 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
// Change the processor name and check if processor name has been changed in Zookeeper.
- cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2");
- cloudConfigInit = cloudConfigInitBuilder.build();
- ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
- admin.addCloudConfig(clusterName, cloudConfigInit);
+ CloudConfig.Builder cloudConfigToUpdateBuilder = new CloudConfig.Builder();
+ cloudConfigToUpdateBuilder.setCloudInfoProcessorName("TestProcessor2");
+ CloudConfig cloudConfigToUpdate = cloudConfigToUpdateBuilder.build();
+ _configAccessor.updateCloudConfig(clusterName, cloudConfigToUpdate);
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
@@ -250,4 +250,50 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
}
+
+ @Test
+ public void testDeleteCloudConfig() throws Exception {
+ ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR);
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestCloudID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE);
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Change the processor name and check if processor name has been changed in Zookeeper.
+ CloudConfig.Builder cloudConfigBuilderToDelete = new CloudConfig.Builder();
+ cloudConfigBuilderToDelete.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigBuilderToDelete.setCloudID("TestCloudID");
+ CloudConfig cloudConfigToDelete = cloudConfigBuilderToDelete.build();
+
+ _configAccessor.deleteCloudConfigFields(clusterName, cloudConfigToDelete);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertNull(cloudConfigFromZk.getCloudID());
+ listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
index 10ba42d..350c9a9 100644
--- a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -55,12 +55,12 @@ public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
// Verify the response from mock http client
AzureCloudInstanceInformation azureCloudInstanceInformation =
processor.parseCloudInstanceInformation(response);
- Assert.assertEquals(azureCloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()), "2");
Assert.assertEquals(
azureCloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()),
- "test-helix");
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()),
+ "faultDomain=2," + "hostname=");
+ Assert.assertEquals(azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()), "test-helix");
Assert.assertEquals(
azureCloudInstanceInformation
.get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 4e6edaf..fa0850a 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -602,6 +602,7 @@ public class ClusterAccessor extends AbstractHelixResource {
return notFound();
}
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
// Here to update cloud config
Command command;
if (commandStr == null || commandStr.isEmpty()) {
@@ -614,22 +615,24 @@ public class ClusterAccessor extends AbstractHelixResource {
}
}
- HelixAdmin admin = getHelixAdmin();
-
ZNRecord record;
+ CloudConfig cloudConfig;
try {
record = toZNRecord(content);
+ cloudConfig = new CloudConfig(record);
} catch (IOException e) {
_logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
return badRequest("Input is not a vaild ZNRecord!");
}
try {
switch (command) {
+ case delete: {
+ configAccessor.deleteCloudConfigFields(clusterId, cloudConfig);
+ }
+ break;
case update: {
try {
- CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
- admin.removeCloudConfig(clusterId);
- admin.addCloudConfig(clusterId, cloudConfig);
+ configAccessor.updateCloudConfig(clusterId, cloudConfig);
} catch (HelixException ex) {
_logger.error("Error in updating a CloudConfig to cluster: " + clusterId, ex);
return badRequest(ex.getMessage());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 81fc708..10ad1c0 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -793,7 +793,7 @@ public class TestClusterAccessor extends AbstractTestClass {
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
-
+
ZNRecord record = new ZNRecord("testZnode");
record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
@@ -810,7 +810,6 @@ public class TestClusterAccessor extends AbstractTestClass {
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNotNull(cloudConfigFromZk);
String urlBase = "clusters/" + clusterName + "/cloudconfig/";
-
delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
@@ -821,7 +820,54 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+
@Test(dependsOnMethods = "testDeleteCloudConfig")
+ public void testPartialDeleteCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+
+ ZNRecord record = new ZNRecord(clusterName);
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), "TestProcessor");
+ _gSetupTool.addCluster(clusterName, true, new CloudConfig.Builder(record).build());
+
+ String urlBase = "clusters/" + clusterName +"/cloudconfig/";
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+ // Read CloudConfig from Zookeeper and make sure it has been created
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNotNull(cloudConfigFromZk);
+
+ record = new ZNRecord(clusterName);
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", Command.delete.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(), CloudProvider.AZURE.name());
+ post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and make sure it has been removed
+ _configAccessor = new ConfigAccessor(ZK_ADDR);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNull(cloudConfigFromZk.getCloudID());
+ Assert.assertNull(cloudConfigFromZk.getCloudProvider());
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(),"TestProcessor");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testPartialDeleteCloudConfig")
public void testUpdateCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
_gSetupTool.addCluster("TestCloud", true);
[helix] 02/10: Add REST API for Cluster Creation with CloudConfig
(#675)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 927cc749a5e35bb66203da7bbe61420cabcf22e5
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Jan 20 01:16:45 2020 -0800
Add REST API for Cluster Creation with CloudConfig (#675)
This commit contains the relevant REST APIs for cluster creation.
This change allows user to create cluster with CloudConfig.
Multiple tests have been added in order to check the functionality of REST calls.
---
.../server/resources/helix/ClusterAccessor.java | 146 +++++++++-
.../helix/rest/server/TestClusterAccessor.java | 299 +++++++++++++++++++++
2 files changed, 438 insertions(+), 7 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index e4f0358..ae31cb5 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -47,6 +47,7 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.ZNRecord;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.manager.zk.client.HelixZkClient;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.HelixConfigScope;
@@ -132,17 +133,35 @@ public class ClusterAccessor extends AbstractHelixResource {
@PUT
@Path("{clusterId}")
public Response createCluster(@PathParam("clusterId") String clusterId,
- @DefaultValue("false") @QueryParam("recreate") String recreate) {
+ @DefaultValue("false") @QueryParam("recreate") String recreate,
+ @DefaultValue("false") @QueryParam("addCloudConfig") String addCloudConfig,
+ String content) {
+
boolean recreateIfExists = Boolean.valueOf(recreate);
+ boolean cloudConfigIncluded = Boolean.valueOf(addCloudConfig);
+
+
ClusterSetup clusterSetup = getClusterSetup();
- try {
- clusterSetup.addCluster(clusterId, recreateIfExists);
- } catch (Exception ex) {
- _logger.error("Failed to create cluster " + clusterId + ", exception: " + ex);
- return serverError(ex);
- }
+ CloudConfig cloudConfig = null;
+ if (cloudConfigIncluded) {
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+ try {
+ cloudConfig = new CloudConfig.Builder(record).build();
+ clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
+ } catch (Exception ex) {
+ _logger.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ }
+ }
+ clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
return created();
}
@@ -514,6 +533,119 @@ public class ClusterAccessor extends AbstractHelixResource {
return ZKUtil.isClusterSetup(cluster, zkClient);
}
+ @PUT
+ @Path("{clusterId}/cloudconfig")
+ public Response addCloudConfig(@PathParam("clusterId") String clusterId, String content) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound("Cluster is not properly setup!");
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+
+ try {
+ CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
+ admin.addCloudConfig(clusterId, cloudConfig);
+ } catch (HelixException ex) {
+ _logger.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ } catch (Exception ex) {
+ _logger.error("Cannot add CloudConfig to cluster: " + clusterId, ex);
+ return serverError(ex);
+ }
+
+ return OK();
+ }
+
+ @GET
+ @Path("{clusterId}/cloudconfig")
+ public Response getCloudConfig(@PathParam("clusterId") String clusterId) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound();
+ }
+
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
+ CloudConfig cloudConfig = configAccessor.getCloudConfig(clusterId);
+
+ if (cloudConfig != null) {
+ return JSONRepresentation(cloudConfig.getRecord());
+ }
+
+ return notFound();
+ }
+
+ @POST
+ @Path("{clusterId}/cloudconfig")
+ public Response updateCloudConfig(@PathParam("clusterId") String clusterId,
+ @QueryParam("command") String commandStr, String content) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound();
+ }
+
+ // Here to update cloud config
+ Command command;
+ if (commandStr == null || commandStr.isEmpty()) {
+ command = Command.update; // Default behavior
+ } else {
+ try {
+ command = getCommand(commandStr);
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+ try {
+ switch (command) {
+ case delete: {
+ admin.removeCloudConfig(clusterId);
+ }
+ break;
+ case update: {
+ try {
+ CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
+ admin.removeCloudConfig(clusterId);
+ admin.addCloudConfig(clusterId, cloudConfig);
+ } catch (HelixException ex) {
+ _logger.error("Error in updating a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ } catch (Exception ex) {
+ _logger.error("Cannot update CloudConfig for cluster: " + clusterId, ex);
+ return serverError(ex);
+ }
+ }
+ break;
+ default:
+ return badRequest("Unsupported command " + commandStr);
+ }
+ } catch (Exception ex) {
+ _logger.error("Failed to " + command + " cloud config, cluster " + clusterId + " new config: "
+ + content + ", Exception: " + ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
/**
* Reads HISTORY ZNode from the metadata store and generates a Map object that contains the
* pertinent history entries depending on the history type.
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index b8d8018..4ea2d79 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -33,6 +33,7 @@ import javax.ws.rs.core.Response;
import com.google.common.collect.ImmutableMap;
import com.sun.research.ws.wadl.HTTPMethods;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
@@ -48,6 +49,8 @@ import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
+import org.apache.helix.model.CloudConfig;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.rest.common.HelixRestNamespace;
import org.apache.helix.rest.server.auditlog.AuditLog;
import org.apache.helix.rest.server.resources.AbstractResource;
@@ -563,6 +566,302 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+
+ @Test(dependsOnMethods = "testActivateSuperCluster")
+ public void testAddClusterWithCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithCloudConfig")
+ public void testAddClusterWithInvalidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider has not been defined. Result of this rest call will be BAD_REQUEST.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig")
+ public void testAddClusterWithInvalidCustomizedCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider is customized. CLOUD_INFO_PROCESSOR_NAME and CLOUD_INFO_SOURCE fields are
+ // required.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCustomizedCloudConfig")
+ public void testAddClusterWithValidCustomizedCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), sourceList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider is customized. CLOUD_INFO_PROCESSOR_NAME and CLOUD_INFO_SOURCE fields are
+ // required.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithValidCustomizedCloudConfig")
+ public void testAddClusterWithCloudConfigDisabledCloud() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), false);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
+
+
+ @Test(dependsOnMethods = "testAddClusterWithCloudConfigDisabledCloud")
+ public void testAddCloudConfigNonExistedCluster() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessor");
+
+ // Not found since the cluster is not setup yet.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.NOT_FOUND.getStatusCode());
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCloudConfigNonExistedCluster")
+ public void testAddCloudConfig() throws Exception {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+
+ // Bad request since Processor has not been defined.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ // Now response should be OK since all fields are set
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+
+ // Now test the getCloudConfig method.
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CloudConfig cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ CloudConfig cloudConfigZk = _configAccessor.getCloudConfig("TestCloud");
+
+ // Check that the CloudConfig from Zk and REST get method are equal
+ Assert.assertEquals(cloudConfigRest, cloudConfigZk);
+
+ // Check the fields individually
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromRest = cloudConfigRest.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromRest.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigRest.getCloudInfoProcessorName(), "TestProcessorName");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCloudConfig")
+ public void testDeleteCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", AbstractResource.Command.delete.name());
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and make sure it has been removed
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ Assert.assertNull(cloudConfigFromZk);
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testDeleteCloudConfig")
+ public void testUpdateCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ // Fist add CloudConfig to the cluster
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Now get the Cloud Config and make sure the information is correct
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CloudConfig cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Now put new information in the ZNRecord
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudIdNew");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", AbstractResource.Command.update.name());
+
+ post(urlBase, map1,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Now get the Cloud Config and make sure the information has been updated
+ body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudIdNew");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromRest = cloudConfigRest.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromRest.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigRest.getCloudInfoProcessorName(), "TestProcessorName");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+
private ClusterConfig getClusterConfigFromRest(String cluster) throws IOException {
String body = get("clusters/" + cluster + "/configs", null, Response.Status.OK.getStatusCode(), true);
[helix] 07/10: Change the cluster creation logic (#872)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit b9d4627eb726e68d39f0edd9273f26a3131d0a63
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800
Change the cluster creation logic (#872)
* Change the cluster creation logic
In this commit, minor modifications have been added which:
1- Update topology information in the Cluster Config
if cloud is enabled for Azure environment.
2- Tests have been change accordingly.
3- The deprecated APIs have been changed in the test to
follow new APIs.
---
.../org/apache/helix/cloud/azure/AzureConstants.java | 6 ++++++
.../main/java/org/apache/helix/tools/ClusterSetup.java | 13 ++++++++++++-
.../java/org/apache/helix/tools/TestClusterSetup.java | 14 +++++++++++---
.../apache/helix/rest/server/TestClusterAccessor.java | 16 +++++++++++-----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+ public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+ public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index d1dc6d3..a8969bc 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -37,12 +37,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
-import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -189,6 +190,16 @@ public class ClusterSetup {
if (cloudConfig != null) {
_admin.addCloudConfig(clusterName, cloudConfig);
+ // If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
+ if (cloudConfig.isCloudEnabled()
+ && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ configAccessor.updateClusterConfig(clusterName, clusterConfig);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 68195bd..0a1664b 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -34,12 +34,14 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
@@ -355,7 +357,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -421,7 +423,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -505,12 +507,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ // Since it is Azure, topology information should have been populated.
+ ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
// Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
Assert.assertNull(listUrlFromZk);
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 4ea2d79..21a2853 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -38,6 +38,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterDistributedController;
@@ -587,11 +588,16 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
}
@Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -662,7 +668,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -692,7 +698,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -751,7 +757,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -795,7 +801,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertNull(cloudConfigFromZk);
[helix] 08/10: Add construction of domain in Helix participant
logic (#876)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 2f6faa9651ce7f6ce6cf1134e1d6d10eb8818ada
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Mar 10 11:10:39 2020 -0700
Add construction of domain in Helix participant logic (#876)
Add the construction of domain field in Azure cloud instance information processor. It will read the topology from AzureConstants, and construct the domain based on the format of the topology.
---
.../helix/cloud/azure/AzureCloudInstanceInformationProcessor.java | 7 ++++++-
.../main/java/org/apache/helix/manager/zk/ParticipantManager.java | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 85e2bb5..464ff55 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -142,8 +142,13 @@ public class AzureCloudInstanceInformationProcessor
String vmName = computeNode.path(INSTANCE_NAME).getTextValue();
String platformFaultDomain = computeNode.path(DOMAIN).getTextValue();
String vmssName = computeNode.path(INSTANCE_SET_NAME).getValueAsText();
+ String azureTopology = AzureConstants.AZURE_TOPOLOGY;
+ String[] parts = azureTopology.trim().split("/");
+ //The hostname will be filled in by each participant
+ String domain = parts[0] + "=" + platformFaultDomain + "," + parts[1] + "=";
+
AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
- builder.setInstanceName(vmName).setFaultDomain(platformFaultDomain)
+ builder.setInstanceName(vmName).setFaultDomain(domain)
.setInstanceSetName(vmssName);
azureCloudInstanceInformation = builder.build();
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 32e0575..7b75c7a 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -178,7 +178,7 @@ public class ParticipantManager {
LOG.info(_instanceName + " is auto-registering cluster: " + _clusterName);
CloudInstanceInformation cloudInstanceInformation = getCloudInstanceInformation();
String domain = cloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name());
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()) + _instanceName;
// Disable the verification for now
/*String cloudIdInRemote = cloudInstanceInformation
[helix] 01/10: Add java API to create cluster with CloudConfig
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 19bbf431f93d3fef46dabbbfcfc511ea3e96f670
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Nov 26 15:26:10 2019 -0800
Add java API to create cluster with CloudConfig
In this commit the below APIs have been added.
1- AddCluster with CloudCOnfig API.
2- addCloudConfig to existing cluster.
3- Update CloudConfig to update existing Cloud Config.
Several tests have been added to test these APIs.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 4 +-
.../src/main/java/org/apache/helix/HelixAdmin.java | 15 ++-
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++++
.../java/org/apache/helix/model/CloudConfig.java | 149 +++++++--------------
.../java/org/apache/helix/tools/ClusterSetup.java | 17 ++-
.../java/org/apache/helix/TestConfigAccessor.java | 66 +++++++--
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 69 ++++++++++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 11 ++
.../apache/helix/model/cloud/TestCloudConfig.java | 60 ++++++---
.../org/apache/helix/tools/TestClusterSetup.java | 88 +++++++++++-
10 files changed, 368 insertions(+), 139 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 264fd3f..97bfb34 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -591,7 +591,7 @@ public class ConfigAccessor {
* @return The instance of {@link CloudConfig}
*/
public CloudConfig getCloudConfig(String clusterName) {
- if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException(
String.format("Failed to get config. cluster: %s is not setup.", clusterName));
}
@@ -604,7 +604,7 @@ public class ConfigAccessor {
return null;
}
- return new CloudConfig(record);
+ return new CloudConfig.Builder(record).build();
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index a11b235..c3dc8b5 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -22,7 +22,7 @@ package org.apache.helix;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
@@ -381,6 +381,19 @@ public interface HelixAdmin {
void dropResource(String clusterName, String resourceName);
/**
+ * Add cloud config to the cluster.
+ * @param clusterName
+ * @param cloudConfig
+ */
+ void addCloudConfig(String clusterName, CloudConfig cloudConfig);
+
+ /**
+ * Remove the Cloud Config for specific cluster
+ * @param clusterName
+ */
+ void removeCloudConfig(String clusterName);
+
+ /**
* Get a list of state model definitions in a cluster
* @param clusterName
* @return
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 0a978e5..9d8d406 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -59,6 +59,7 @@ import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.manager.zk.client.SharedZkClientFactory;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -1026,6 +1027,33 @@ public class ZKHelixAdmin implements HelixAdmin {
}
@Override
+ public void addCloudConfig(String clusterName, CloudConfig cloudConfig) {
+ logger.info("Add CloudConfig to cluster {}, CloudConfig is {}.", clusterName,
+ cloudConfig.toString());
+
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("cluster " + clusterName + " is not setup yet");
+ }
+
+ CloudConfig.Builder builder = new CloudConfig.Builder(cloudConfig);
+ CloudConfig cloudConfigBuilder = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfigBuilder);
+ }
+
+ @Override
+ public void removeCloudConfig(String clusterName) {
+ logger.info("Remove Cloud Config for cluster {}.", clusterName);
+ HelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.removeProperty(keyBuilder.cloudConfig());
+ }
+
+ @Override
public List<String> getStateModelDefs(String clusterName) {
return _zkClient.getChildren(PropertyPathBuilder.stateModelDef(clusterName));
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index c8ab6eb..f6c279c 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -30,6 +30,9 @@ import org.apache.helix.cloud.constants.CloudProvider;
* Cloud configurations
*/
public class CloudConfig extends HelixProperty {
+
+ public static final String CLOUD_CONFIG_KW = "CloudConfig";
+
/**
* Configurable characteristics of a cloud.
* NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
@@ -52,39 +55,22 @@ public class CloudConfig extends HelixProperty {
/**
* Instantiate the CloudConfig for the cloud
- * @param cluster
*/
- public CloudConfig(String cluster) {
- super(cluster);
+ private CloudConfig() {
+ super(CLOUD_CONFIG_KW);
}
/**
- * Instantiate with a pre-populated record
- * @param record a ZNRecord corresponding to a cloud configuration
+ * The constructor from the ZNRecord.
+ * @param record
*/
- public CloudConfig(ZNRecord record) {
- super(record);
+ private CloudConfig(ZNRecord record) {
+ super(CLOUD_CONFIG_KW);
+ _record.setSimpleFields(record.getSimpleFields());
+ _record.setListFields(record.getListFields());
+ _record.setMapFields(record.getMapFields());
}
- /**
- * Instantiate the config using each field individually.
- * Users should use CloudConfig.Builder to create CloudConfig.
- * @param cluster
- * @param enabled
- * @param cloudID
- */
- public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
- List<String> cloudInfoSource, String cloudProcessorName) {
- super(cluster);
- _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
- _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
- _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
- if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
- _record
- .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
- _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
- }
- }
/**
* Enable/Disable the CLOUD_ENABLED field.
@@ -135,15 +121,6 @@ public class CloudConfig extends HelixProperty {
}
/**
- * Set the CLOUD_INFO_PROCESSOR_NAME field.
- * @param cloudInfoProcessorName
- */
- public void setCloudInfoFProcessorName(String cloudInfoProcessorName) {
- _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
- cloudInfoProcessorName);
- }
-
- /**
* Get the CLOUD_INFO_PROCESSOR_NAME field.
* @return CLOUD_INFO_PROCESSOR_NAME field.
*/
@@ -152,14 +129,6 @@ public class CloudConfig extends HelixProperty {
}
/**
- * Set the CLOUD_PROVIDER field.
- * @param cloudProvider
- */
- public void setCloudProvider(CloudProvider cloudProvider) {
- _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
- }
-
- /**
* Get the CLOUD_PROVIDER field.
* @return CLOUD_PROVIDER field.
*/
@@ -167,32 +136,28 @@ public class CloudConfig extends HelixProperty {
return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
}
+
public static class Builder {
- private String _clusterName = null;
- private CloudProvider _cloudProvider;
- private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED;
- private String _cloudID;
- private List<String> _cloudInfoSources;
- private String _cloudInfoProcessorName;
+ private ZNRecord _record;
public CloudConfig build() {
validate();
- return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID,
- _cloudInfoSources, _cloudInfoProcessorName);
+ return new CloudConfig(_record);
}
/**
* Default constructor
*/
public Builder() {
+ _record = new ZNRecord(CLOUD_CONFIG_KW);
}
/**
- * Constructor with Cluster Name as input
- * @param clusterName
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a cloud configuration
*/
- public Builder(String clusterName) {
- _clusterName = clusterName;
+ public Builder(ZNRecord record) {
+ _record = record;
}
/**
@@ -200,89 +165,75 @@ public class CloudConfig extends HelixProperty {
* @param cloudConfig
*/
public Builder(CloudConfig cloudConfig) {
- _cloudEnabled = cloudConfig.isCloudEnabled();
- _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider());
- _cloudID = cloudConfig.getCloudID();
- _cloudInfoSources = cloudConfig.getCloudInfoSources();
- _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName();
- }
-
- public Builder setClusterName(String v) {
- _clusterName = v;
- return this;
+ _record = cloudConfig.getRecord();
}
public Builder setCloudEnabled(boolean isEnabled) {
- _cloudEnabled = isEnabled;
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), isEnabled);
return this;
}
public Builder setCloudProvider(CloudProvider cloudProvider) {
- _cloudProvider = cloudProvider;
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
return this;
}
- public Builder setCloudID(String v) {
- _cloudID = v;
+ public Builder setCloudID(String cloudID) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
return this;
}
- public Builder setCloudInfoSources(List<String> v) {
- _cloudInfoSources = v;
+ public Builder setCloudInfoSources(List<String> cloudInfoSources) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources);
return this;
}
- public Builder addCloudInfoSource(String v) {
- if (_cloudInfoSources == null) {
- _cloudInfoSources = new ArrayList<String>();
+ public Builder addCloudInfoSource(String cloudInfoSource) {
+ if (_record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()) == null) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), new ArrayList<String>());
}
- _cloudInfoSources.add(v);
+ List<String> cloudInfoSourcesList = _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
+ cloudInfoSourcesList.add(cloudInfoSource);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSourcesList);
return this;
}
- public Builder setCloudInfoProcessorName(String v) {
- _cloudInfoProcessorName = v;
+ public Builder setCloudInfoProcessorName(String cloudInfoProcessorName) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ cloudInfoProcessorName);
return this;
}
- public String getClusterName() {
- return _clusterName;
- }
-
- public CloudProvider getCloudProvider() {
- return _cloudProvider;
+ public String getCloudProvider() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
}
public boolean getCloudEnabled() {
- return _cloudEnabled;
+ return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(),
+ DEFAULT_CLOUD_ENABLED);
}
public String getCloudID() {
- return _cloudID;
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name());
}
public List<String> getCloudInfoSources() {
- return _cloudInfoSources;
+ return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
}
public String getCloudInfoProcessorName() {
- return _cloudInfoProcessorName;
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name());
}
private void validate() {
- if (_cloudEnabled) {
- if (_cloudID == null) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. The CloudID is missing from the config.");
- }
- if (_cloudProvider == null) {
+ if (this.getCloudProvider() == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
+ if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
+ || this.getCloudInfoSources().size() == 0) {
throw new HelixException(
- "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
- } else if (_cloudProvider == CloudProvider.CUSTOMIZED) {
- if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
- }
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
}
}
}
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 5d5f864..d1dc6d3 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -41,7 +41,9 @@ import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
+import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
@@ -49,6 +51,7 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.manager.zk.client.SharedZkClientFactory;
import org.apache.helix.model.BuiltInStateModelDefinitions;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -176,15 +179,23 @@ public class ClusterSetup {
}
}
- public void addCluster(String clusterName, boolean overwritePrevious) {
+ public void addCluster(String clusterName, boolean overwritePrevious, CloudConfig cloudConfig)
+ throws HelixException {
_admin.addCluster(clusterName, overwritePrevious);
-
for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) {
addStateModelDef(clusterName, def.getStateModelDefinition().getId(),
- def.getStateModelDefinition(), overwritePrevious);
+ def.getStateModelDefinition(), overwritePrevious);
+ }
+
+ if (cloudConfig != null) {
+ _admin.addCloudConfig(clusterName, cloudConfig);
}
}
+ public void addCluster(String clusterName, boolean overwritePrevious) {
+ addCluster(clusterName, overwritePrevious, null);
+ }
+
public void activateCluster(String clusterName, String grandCluster, boolean enable) {
if (enable) {
_admin.addClusterToGrandCluster(clusterName, grandCluster);
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index 0fa05c3..e62bf81 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -19,14 +19,18 @@ package org.apache.helix;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.builder.ConfigScopeBuilder;
+import org.apache.helix.tools.ClusterSetup;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -68,9 +72,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(resourceConfigValue, "resourceConfigValue");
// partition scope config
- ConfigScope partitionScope =
- new ConfigScopeBuilder().forCluster(clusterName).forResource("testResource")
- .forPartition("testPartition").build();
+ ConfigScope partitionScope = new ConfigScopeBuilder().forCluster(clusterName)
+ .forResource("testResource").forPartition("testPartition").build();
configAccessor.set(partitionScope, "partitionConfigKey", "partitionConfigValue");
String partitionConfigValue = configAccessor.get(partitionScope, "partitionConfigKey");
Assert.assertEquals(partitionConfigValue, "partitionConfigValue");
@@ -113,9 +116,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
"should be [HELIX_ENABLED, HELIX_ENABLED_TIMESTAMP, HELIX_HOST, HELIX_PORT, participantConfigKey]");
Assert.assertEquals(keys.get(4), "participantConfigKey");
- keys =
- configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource",
- "testPartition");
+ keys = configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource",
+ "testPartition");
Assert.assertEquals(keys.size(), 1, "should be [partitionConfigKey]");
Assert.assertEquals(keys.get(0), "partitionConfigKey");
@@ -183,8 +185,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
try {
configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue");
- Assert
- .fail("Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet");
+ Assert.fail(
+ "Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet");
} catch (HelixException e) {
// OK
}
@@ -193,8 +195,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
try {
configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue");
} catch (Exception e) {
- Assert
- .fail("Except succeed to set participant-config because participant: localhost_12918 has been added to cluster");
+ Assert.fail(
+ "Except succeed to set participant-config because participant: localhost_12918 has been added to cluster");
}
String participantConfigValue = configAccessor.get(participantScope, "participantConfigKey");
@@ -204,4 +206,48 @@ public class TestConfigAccessor extends ZkUnitTestBase {
configAccessor.close();
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
+
+ @Test
+ public void testUpdateCloudConfig() throws Exception {
+ ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR);
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestCloudID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+
+ // Change the processor name and check if processor name has been changed in Zookeeper.
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2");
+ cloudConfigInit = cloudConfigInitBuilder.build();
+ ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCloudConfig(clusterName, cloudConfigInit);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index 20acef4..69fa558 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import org.apache.helix.BaseDataAccessor;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
@@ -39,7 +40,9 @@ import org.apache.helix.PropertyType;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.examples.MasterSlaveStateModelFactory;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -506,4 +509,70 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
.getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
2);
}
+
+ @Test
+ public void testAddCloudConfig() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+
+ CloudConfig.Builder builder = new CloudConfig.Builder();
+ builder.setCloudEnabled(true);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL");
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+
+ admin.addCloudConfig(clusterName, cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+
+ @Test
+ public void testRemoveCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+
+ CloudConfig.Builder builder = new CloudConfig.Builder();
+ builder.setCloudEnabled(true);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL");
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+
+ admin.addCloudConfig(clusterName, cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+
+ // Remove Cloud Config and make sure it has been removed from Zookeeper
+ admin.removeCloudConfig(clusterName);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNull(cloudConfigFromZk);
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index b299ab1..76e219b 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -31,6 +31,7 @@ import org.apache.helix.HelixManager;
import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
import org.apache.helix.ZNRecord;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ConstraintItem;
@@ -313,6 +314,16 @@ public class MockHelixAdmin implements HelixAdmin {
}
+ @Override
+ public void addCloudConfig(String clusterName, CloudConfig cloudConfig) {
+
+ }
+
+ @Override
+ public void removeCloudConfig(String clusterName) {
+
+ }
+
@Override public List<String> getStateModelDefs(String clusterName) {
return null;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
index ee83011..01945de 100644
--- a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
@@ -57,20 +57,22 @@ public class TestCloudConfig extends ZkUnitTestBase {
}
@Test(dependsOnMethods = "testCloudConfigNull")
- public void testCloudConfig() {
+ public void testCloudConfig() throws Exception {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
// Create dummy CloudConfig object
- CloudConfig cloudConfig = new CloudConfig(clusterName);
- cloudConfig.setCloudEnabled(true);
- cloudConfig.setCloudProvider(CloudProvider.AZURE);
- cloudConfig.setCloudID("TestID");
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
List<String> infoURL = new ArrayList<String>();
infoURL.add("TestURL");
- cloudConfig.setCloudInfoSource(infoURL);
- cloudConfig.setCloudInfoFProcessorName("TestProcessor");
+ cloudConfigBuilder.setCloudInfoSources(infoURL);
+ cloudConfigBuilder.setCloudInfoProcessorName("TestProcessor");
+
+ CloudConfig cloudConfig = cloudConfigBuilder.build();
// Write the CloudConfig to Zookeeper
ZKHelixDataAccessor accessor =
@@ -84,8 +86,8 @@ public class TestCloudConfig extends ZkUnitTestBase {
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+
Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1);
- Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL");
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
}
@@ -93,7 +95,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
public void testUnverifiedCloudConfigBuilder() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
// Verify will fail because cloudID has net been defined.
CloudConfig cloudConfig = builder.build();
@@ -103,7 +105,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
public void testUnverifiedCloudConfigBuilderEmptySources() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -115,9 +117,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
@Test(expectedExceptions = HelixException.class)
public void testUnverifiedCloudConfigBuilderWithoutProcessor() {
- String className = getShortClassName();
- String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -132,7 +132,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -142,8 +142,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
// Check builder getter methods
Assert.assertTrue(builder.getCloudEnabled());
- Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED);
- Assert.assertEquals(builder.getClusterName(), clusterName);
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
Assert.assertEquals(builder.getCloudID(), "TestID");
List<String> listUrlFromBuilder = builder.getCloudInfoSources();
Assert.assertEquals(listUrlFromBuilder.size(), 2);
@@ -175,15 +174,14 @@ public class TestCloudConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.AZURE);
builder.setCloudID("TestID");
- builder.setCloudInfoProcessorName("TestProcessor");
// Check builder getter methods
Assert.assertTrue(builder.getCloudEnabled());
- Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE);
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE.name());
CloudConfig cloudConfig = builder.build();
@@ -198,7 +196,29 @@ public class TestCloudConfig extends ZkUnitTestBase {
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
- // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null.
+ // Since user does not set the CloudInfoProcessorName, this field will be null.
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+
+ // Checking the set method in CloudConfig
+ cloudConfig.setCloudEnabled(false);
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
+
+ cloudConfig.setCloudEnabled(true);
+ cloudConfig.setCloudID("TestID2");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL0");
+ sourceList.add("TestURL1");
+ cloudConfig.setCloudInfoSource(sourceList);
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID2");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromZk.get(1), "TestURL1");
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index df3f341..68195bd 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -19,10 +19,13 @@ package org.apache.helix.tools;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.List;
import org.apache.helix.BaseDataAccessor;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey;
@@ -31,15 +34,15 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.AssertJUnit;
import org.testng.annotations.AfterClass;
@@ -48,8 +51,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class TestClusterSetup extends ZkUnitTestBase {
- private static Logger LOG = LoggerFactory.getLogger(TestClusterSetup.class);
-
protected static final String CLUSTER_NAME = "TestClusterSetup";
protected static final String TEST_DB = "TestDB";
protected static final String INSTANCE_PREFIX = "instance_";
@@ -436,4 +437,83 @@ public class TestClusterSetup extends ZkUnitTestBase {
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testAddClusterWithInvalidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+
+ // Since setCloudInfoProcessorName is missing, this add cluster call will throw an exception
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig")
+ public void testAddClusterWithValidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessorName");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
+
+
+ @Test(dependsOnMethods = "testAddClusterWithValidCloudConfig")
+ public void testAddClusterAzureProvider() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestID");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+
+ // Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
+ Assert.assertNull(listUrlFromZk);
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
}
[helix] 05/10: Modify participant manager to add cluster auto
registration logic (#695)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit cb672d7b28a89c1c19083450b145bcd51179b618
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Fri Feb 7 15:41:36 2020 -0800
Modify participant manager to add cluster auto registration logic (#695)
Modify participant logic to add logic for instance auto registration to a Helix cluster.
Auto registration differs with existing auto join in that auto registration will retrieve the fault domain information for the instance and populate it under instanceConfig. With auto registration on, user does not need to manually populate instance information in Zookeeper.
---
.../helix/manager/zk/ParticipantManager.java | 100 ++++++++++++++++-----
.../main/java/org/apache/helix/util/HelixUtil.java | 19 ++++
.../manager/MockParticipantManager.java | 10 ++-
3 files changed, 107 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 9c5d602..32e0575 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -20,6 +20,8 @@ package org.apache.helix.manager.zk;
*/
import java.lang.management.ManagementFactory;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -31,6 +33,7 @@ import org.apache.helix.AccessOption;
import org.apache.helix.BaseDataAccessor;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixCloudProperty;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerProperty;
@@ -40,6 +43,8 @@ import org.apache.helix.PreConnectCallback;
import org.apache.helix.PropertyKey;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZNRecordBucketizer;
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.messaging.DefaultMessagingService;
import org.apache.helix.model.CurrentState;
@@ -52,6 +57,7 @@ import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.helix.participant.StateMachineEngine;
import org.apache.helix.participant.statemachine.ScheduledTaskStateModelFactory;
+import org.apache.helix.util.HelixUtil;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +67,7 @@ import org.slf4j.LoggerFactory;
*/
public class ParticipantManager {
private static Logger LOG = LoggerFactory.getLogger(ParticipantManager.class);
+ private static final String CLOUD_PROCESSOR_PATH_PREFIX = "org.apache.helix.cloud.";
final HelixZkClient _zkclient;
final HelixManager _manager;
@@ -107,7 +114,7 @@ public class ParticipantManager {
}
/**
- * Handle new session for a participang.
+ * Handle new session for a participant.
* @throws Exception
*/
public void handleNewSession() throws Exception {
@@ -132,18 +139,31 @@ public class ParticipantManager {
}
private void joinCluster() {
- // Read cluster config and see if instance can auto join the cluster
+ // Read cluster config and see if an instance can auto join or auto register to the cluster
boolean autoJoin = false;
+ boolean autoRegistration = false;
+
+ // Read "allowParticipantAutoJoin" flag to see if an instance can auto join to the cluster
try {
- HelixConfigScope scope =
- new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(
- _manager.getClusterName()).build();
- autoJoin =
- Boolean.parseBoolean(_configAccessor.get(scope,
- ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN));
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
+ .forCluster(_manager.getClusterName()).build();
+ autoJoin = Boolean
+ .parseBoolean(_configAccessor.get(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN));
LOG.info("instance: " + _instanceName + " auto-joining " + _clusterName + " is " + autoJoin);
} catch (Exception e) {
- // autoJoin is false
+ LOG.info("auto join is false for cluster" + _clusterName);
+ }
+
+ // Read cloud config and see if an instance can auto register to the cluster
+ // Difference between auto join and auto registration is that the latter will also populate the
+ // domain information in instance config
+ try {
+ autoRegistration =
+ Boolean.valueOf(_helixManagerProperty.getHelixCloudProperty().getCloudEnabled());
+ LOG.info("instance: " + _instanceName + " auto-register " + _clusterName + " is "
+ + autoRegistration);
+ } catch (Exception e) {
+ LOG.info("auto registration is false for cluster" + _clusterName);
}
if (!ZKUtil.isInstanceSetup(_zkclient, _clusterName, _instanceName, _instanceType)) {
@@ -151,23 +171,61 @@ public class ParticipantManager {
throw new HelixException("Initial cluster structure is not set up for instance: "
+ _instanceName + ", instanceType: " + _instanceType);
} else {
- LOG.info(_instanceName + " is auto-joining cluster: " + _clusterName);
- InstanceConfig instanceConfig = new InstanceConfig(_instanceName);
- String hostName = _instanceName;
- String port = "";
- int lastPos = _instanceName.lastIndexOf("_");
- if (lastPos > 0) {
- hostName = _instanceName.substring(0, lastPos);
- port = _instanceName.substring(lastPos + 1);
+ if (!autoRegistration) {
+ LOG.info(_instanceName + " is auto-joining cluster: " + _clusterName);
+ _helixAdmin.addInstance(_clusterName, HelixUtil.composeInstanceConfig(_instanceName));
+ } else {
+ LOG.info(_instanceName + " is auto-registering cluster: " + _clusterName);
+ CloudInstanceInformation cloudInstanceInformation = getCloudInstanceInformation();
+ String domain = cloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name());
+
+ // Disable the verification for now
+ /*String cloudIdInRemote = cloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name());
+ String cloudIdInConfig = _configAccessor.getCloudConfig(_clusterName).getCloudID();
+
+ // validate that the instance is auto registering to the correct cluster
+ if (!cloudIdInRemote.equals(cloudIdInConfig)) {
+ throw new IllegalArgumentException(String.format(
+ "cloudId in config: %s is not consistent with cloudId from remote: %s. The instance is auto registering to a wrong cluster.",
+ cloudIdInConfig, cloudIdInRemote));
+ }*/
+
+ InstanceConfig instanceConfig = HelixUtil.composeInstanceConfig(_instanceName);
+ instanceConfig.setDomain(domain);
+ _helixAdmin.addInstance(_clusterName, instanceConfig);
}
- instanceConfig.setHostName(hostName);
- instanceConfig.setPort(port);
- instanceConfig.setInstanceEnabled(true);
- _helixAdmin.addInstance(_clusterName, instanceConfig);
}
}
}
+ private CloudInstanceInformation getCloudInstanceInformation() {
+ String cloudInstanceInformationProcessorName =
+ _helixManagerProperty.getHelixCloudProperty().getCloudInfoProcessorName();
+ try {
+ // fetch cloud instance information for the instance
+ String cloudInstanceInformationProcessorClassName = CLOUD_PROCESSOR_PATH_PREFIX
+ + _helixManagerProperty.getHelixCloudProperty().getCloudProvider().toLowerCase() + "."
+ + cloudInstanceInformationProcessorName;
+ Class processorClass = Class.forName(cloudInstanceInformationProcessorClassName);
+ Constructor constructor = processorClass.getConstructor(HelixCloudProperty.class);
+ CloudInstanceInformationProcessor processor = (CloudInstanceInformationProcessor) constructor
+ .newInstance(_helixManagerProperty.getHelixCloudProperty());
+ List<String> responses = processor.fetchCloudInstanceInformation();
+
+ // parse cloud instance information for the participant
+ CloudInstanceInformation cloudInstanceInformation =
+ processor.parseCloudInstanceInformation(responses);
+ return cloudInstanceInformation;
+ } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException
+ | IllegalAccessException | InvocationTargetException ex) {
+ throw new HelixException(
+ "Failed to create a new instance for the class: " + cloudInstanceInformationProcessorName,
+ ex);
+ }
+ }
+
private void createLiveInstance() {
String liveInstancePath = _keyBuilder.liveInstance(_instanceName).getPath();
LiveInstance liveInstance = new LiveInstance(_instanceName);
diff --git a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
index a31c3fe..3b41b83 100644
--- a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
@@ -290,4 +290,23 @@ public final class HelixUtil {
return propertyDefaultValue;
}
+ /**
+ * Compose the config for an instance
+ * @param instanceName
+ * @return InstanceConfig
+ */
+ public static InstanceConfig composeInstanceConfig(String instanceName) {
+ InstanceConfig instanceConfig = new InstanceConfig(instanceName);
+ String hostName = instanceName;
+ String port = "";
+ int lastPos = instanceName.lastIndexOf("_");
+ if (lastPos > 0) {
+ hostName = instanceName.substring(0, lastPos);
+ port = instanceName.substring(lastPos + 1);
+ }
+ instanceConfig.setHostName(hostName);
+ instanceConfig.setPort(port);
+ instanceConfig.setInstanceEnabled(true);
+ return instanceConfig;
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
index ed4612e..e2623cf 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
@@ -22,6 +22,7 @@ package org.apache.helix.integration.manager;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import org.apache.helix.HelixCloudProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.manager.zk.CallbackHandler;
import org.apache.helix.manager.zk.ZKHelixManager;
@@ -48,6 +49,7 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
protected MockMSModelFactory _msModelFactory;
protected DummyLeaderStandbyStateModelFactory _lsModelFactory;
protected DummyOnlineOfflineStateModelFactory _ofModelFactory;
+ protected HelixCloudProperty _helixCloudProperty;
public MockParticipantManager(String zkAddr, String clusterName, String instanceName) {
this(zkAddr, clusterName, instanceName, 10);
@@ -55,11 +57,17 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
public MockParticipantManager(String zkAddr, String clusterName, String instanceName,
int transDelay) {
+ this(zkAddr, clusterName, instanceName, transDelay, null);
+ }
+
+ public MockParticipantManager(String zkAddr, String clusterName, String instanceName,
+ int transDelay, HelixCloudProperty helixCloudProperty) {
super(clusterName, instanceName, InstanceType.PARTICIPANT, zkAddr);
_transDelay = transDelay;
_msModelFactory = new MockMSModelFactory(null);
_lsModelFactory = new DummyLeaderStandbyStateModelFactory(_transDelay);
_ofModelFactory = new DummyOnlineOfflineStateModelFactory(_transDelay);
+ _helixCloudProperty = helixCloudProperty;
}
public void setTransition(MockTransition transition) {
@@ -136,4 +144,4 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
public List<CallbackHandler> getHandlers() {
return _handlers;
}
-}
+}
\ No newline at end of file