You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2020/04/14 02:20:58 UTC
[helix] 07/10: Change the cluster creation logic (#872)
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch helix-cloud
in repository https://gitbox.apache.org/repos/asf/helix.git
commit b9d4627eb726e68d39f0edd9273f26a3131d0a63
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800
Change the cluster creation logic (#872)
* Change the cluster creation logic
In this commit, minor modifications have been added which:
1- Update topology information in the Cluster Config
if cloud is enabled for Azure environment.
2- Tests have been change accordingly.
3- The deprecated APIs have been changed in the test to
follow new APIs.
---
.../org/apache/helix/cloud/azure/AzureConstants.java | 6 ++++++
.../main/java/org/apache/helix/tools/ClusterSetup.java | 13 ++++++++++++-
.../java/org/apache/helix/tools/TestClusterSetup.java | 14 +++++++++++---
.../apache/helix/rest/server/TestClusterAccessor.java | 16 +++++++++++-----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+ public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+ public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index d1dc6d3..a8969bc 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -37,12 +37,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
-import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -189,6 +190,16 @@ public class ClusterSetup {
if (cloudConfig != null) {
_admin.addCloudConfig(clusterName, cloudConfig);
+ // If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
+ if (cloudConfig.isCloudEnabled()
+ && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ configAccessor.updateClusterConfig(clusterName, clusterConfig);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 68195bd..0a1664b 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -34,12 +34,14 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
@@ -355,7 +357,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -421,7 +423,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -505,12 +507,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ // Since it is Azure, topology information should have been populated.
+ ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
// Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
Assert.assertNull(listUrlFromZk);
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 4ea2d79..21a2853 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -38,6 +38,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterDistributedController;
@@ -587,11 +588,16 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
}
@Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -662,7 +668,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -692,7 +698,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -751,7 +757,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -795,7 +801,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertNull(cloudConfigFromZk);