You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ji...@apache.org on 2020/11/18 19:04:18 UTC
[helix] 09/18: Change the cluster creation logic (#872)
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 2ee9d3c0f3086b02bb8ee35b6ec352990d4be8d3
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800
Change the cluster creation logic (#872)
* Change the cluster creation logic
In this commit, minor modifications have been added which:
1- Update topology information in the Cluster Config
if cloud is enabled for Azure environment.
2- Tests have been change accordingly.
3- The deprecated APIs have been changed in the test to
follow new APIs.
---
.../org/apache/helix/cloud/azure/AzureConstants.java | 6 ++++++
.../main/java/org/apache/helix/tools/ClusterSetup.java | 13 ++++++++++++-
.../java/org/apache/helix/tools/TestClusterSetup.java | 14 +++++++++++---
.../apache/helix/rest/server/TestClusterAccessor.java | 16 +++++++++++-----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+ public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+ public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 27d8c88..9663223 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -36,12 +36,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
-import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -171,6 +172,16 @@ public class ClusterSetup {
if (cloudConfig != null) {
_admin.addCloudConfig(clusterName, cloudConfig);
+ // If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
+ if (cloudConfig.isCloudEnabled()
+ && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ configAccessor.updateClusterConfig(clusterName, clusterConfig);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 6a6d48d..1772521 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -33,12 +33,14 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
@@ -354,7 +356,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -420,7 +422,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -504,12 +506,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ // Since it is Azure, topology information should have been populated.
+ ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
// Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
Assert.assertNull(listUrlFromZk);
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 2df28bd..8102db7 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -38,6 +38,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterDistributedController;
@@ -590,11 +591,16 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
}
@Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -665,7 +671,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -695,7 +701,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -754,7 +760,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -798,7 +804,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertNull(cloudConfigFromZk);