You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2020/04/07 01:20:38 UTC
[helix] 09/12: Change the cluster creation logic (#872)
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 912e7943e3c2a6e70f2427bc3295e52828f37b73
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800
Change the cluster creation logic (#872)
* Change the cluster creation logic
In this commit, minor modifications have been added which:
1- Update topology information in the Cluster Config
if cloud is enabled for Azure environment.
2- Tests have been change accordingly.
3- The deprecated APIs have been changed in the test to
follow new APIs.
---
.../org/apache/helix/cloud/azure/AzureConstants.java | 6 ++++++
.../main/java/org/apache/helix/tools/ClusterSetup.java | 13 ++++++++++++-
.../java/org/apache/helix/tools/TestClusterSetup.java | 14 +++++++++++---
.../apache/helix/rest/server/TestClusterAccessor.java | 16 +++++++++++-----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+ public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+ public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 1d51a74..881a8f6 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -36,12 +36,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
-import org.apache.helix.SystemPropertyKeys;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -189,6 +190,16 @@ public class ClusterSetup {
if (cloudConfig != null) {
_admin.addCloudConfig(clusterName, cloudConfig);
+ // If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
+ if (cloudConfig.isCloudEnabled()
+ && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ configAccessor.updateClusterConfig(clusterName, clusterConfig);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index bd5449f..ee5ae07 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -34,12 +34,14 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
@@ -355,7 +357,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -421,7 +423,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -505,12 +507,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ // Since it is Azure, topology information should have been populated.
+ ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
// Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
Assert.assertNull(listUrlFromZk);
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 09406e1..6eae926 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -39,6 +39,7 @@ import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.model.RESTConfig;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
@@ -665,11 +666,16 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
}
@Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -740,7 +746,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -770,7 +776,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -829,7 +835,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -873,7 +879,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertNull(cloudConfigFromZk);