You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ji...@apache.org on 2020/11/18 19:04:09 UTC
[helix] branch helix-0.9.x updated (615af36 -> 681fdbb)
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a change to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git.
from 615af36 Update ivy files.
new 6800dc3 Add CloudConfig code
new 4286e2f add Helix cloud interface and implementation skeleton methods
new eb6c51a Add java API to create cluster with CloudConfig
new ffe8eea Add REST API for Cluster Creation with CloudConfig (#675)
new bc3dd83 Add Helix properties factory and class (#653)
new da32e4e Implement Azure cloud instance information processor (#698)
new ffe55cc Modify participant manager to add cluster auto registration logic (#695)
new c80fdb3 add one more test for auto registration (#806)
new 2ee9d3c Change the cluster creation logic (#872)
new 3a11313 Add construction of domain in Helix participant logic (#876)
new 90c9e25 Change the REST call for delete CloudConfig (#882)
new 980fbd1 Add REST and JAVA API to modify existing cloudconfig (#898)
new acebdae Fix ClusterAccessor::createCluster wrt CloudConfig (#937)
new ef5722e Change TestInstanceAutoJoin to adapt to cloud environment (#1265)
new 35bbc7c Minor fix to add participant auto registration
new bdcb37a Add TrieClusterTopology for retrieving hierarchical topology (#1307)
new 790e158 Add REST API for cluster topology (#1416)
new 681fdbb Return "name" field as VM name in Azure environment (#1340)
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
helix-core/pom.xml | 5 +
.../main/java/org/apache/helix/ConfigAccessor.java | 65 ++-
.../src/main/java/org/apache/helix/HelixAdmin.java | 21 +
.../java/org/apache/helix/HelixCloudProperty.java | 183 ++++++++
.../java/org/apache/helix/HelixManagerFactory.java | 2 +-
.../org/apache/helix/HelixManagerProperty.java | 74 ++++
.../org/apache/helix/HelixPropertyFactory.java | 87 ++++
.../main/java/org/apache/helix/PropertyKey.java | 11 +
.../java/org/apache/helix/SystemPropertyKeys.java | 8 +
.../cloud/CloudInstanceInformation.java} | 21 +-
.../CloudInstanceInformationProcessor.java} | 23 +-
.../apache/helix/api/topology/ClusterTopology.java | 192 +++++++++
.../cloud/azure/AzureCloudInstanceInformation.java | 72 ++++
.../AzureCloudInstanceInformationProcessor.java | 160 +++++++
.../apache/helix/cloud/azure/AzureConstants.java | 6 +
.../constants/CloudProvider.java} | 7 +-
.../helix/manager/zk/ParticipantManager.java | 110 ++++-
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 48 +++
.../apache/helix/manager/zk/ZKHelixManager.java | 17 +-
.../java/org/apache/helix/model/CloudConfig.java | 263 ++++++++++++
.../java/org/apache/helix/model/ClusterTrie.java | 227 ++++++++++
.../org/apache/helix/model/HelixConfigScope.java | 6 +-
.../main/java/org/apache/helix/model/TrieNode.java | 49 ++-
.../model/builder/HelixConfigScopeBuilder.java | 3 +
.../java/org/apache/helix/tools/ClusterSetup.java | 28 +-
.../main/java/org/apache/helix/util/HelixUtil.java | 19 +
.../apache/helix/util/InstanceValidationUtil.java | 14 +-
.../dummy.sh => resources/azure-cloud.properties} | 12 +-
...version.properties => helix-manager.properties} | 5 +-
.../java/org/apache/helix/TestConfigAccessor.java | 112 ++++-
.../org/apache/helix/cloud/MockHttpClient.java | 53 +++
...TestAzureCloudInstanceInformationProcessor.java | 69 +++
.../manager/MockParticipantManager.java | 10 +-
.../paticipant/TestInstanceAutoJoin.java | 69 ++-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 145 ++++++-
.../java/org/apache/helix/mock/MockHelixAdmin.java | 17 +
.../org/apache/helix/model/TestClusterTrie.java | 141 +++++++
.../apache/helix/model/cloud/TestCloudConfig.java | 224 ++++++++++
.../org/apache/helix/tools/TestClusterSetup.java | 96 ++++-
helix-core/src/test/resources/AzureResponse.json | 104 +++++
helix-rest/pom.xml | 5 -
.../server/resources/helix/ClusterAccessor.java | 176 +++++++-
.../helix/rest/server/AbstractTestClass.java | 5 +-
.../helix/rest/server/TestClusterAccessor.java | 468 ++++++++++++++++++++-
44 files changed, 3307 insertions(+), 125 deletions(-)
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
copy helix-core/src/main/java/org/apache/helix/{HelixTimerTask.java => api/cloud/CloudInstanceInformation.java} (62%)
copy helix-core/src/main/java/org/apache/helix/api/{listeners/IdealStateChangeListener.java => cloud/CloudInstanceInformationProcessor.java} (57%)
create mode 100644 helix-core/src/main/java/org/apache/helix/api/topology/ClusterTopology.java
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
copy helix-core/src/main/java/org/apache/helix/{controller/pipeline/StageContext.java => cloud/constants/CloudProvider.java} (89%)
create mode 100644 helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
create mode 100644 helix-core/src/main/java/org/apache/helix/model/ClusterTrie.java
copy helix-agent/src/main/java/org/apache/helix/agent/CommandAttribute.java => helix-core/src/main/java/org/apache/helix/model/TrieNode.java (53%)
copy helix-core/src/main/{scripts/integration-test/testcases/dummy.sh => resources/azure-cloud.properties} (68%)
mode change 100755 => 100644
copy helix-core/src/main/resources/{cluster-manager-version.properties => helix-manager.properties} (88%)
create mode 100644 helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
create mode 100644 helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
create mode 100644 helix-core/src/test/java/org/apache/helix/model/TestClusterTrie.java
create mode 100644 helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
create mode 100644 helix-core/src/test/resources/AzureResponse.json
[helix] 10/18: Add construction of domain in Helix participant
logic (#876)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 3a113138a3323432dcb9010c0a0f5f3aa778fde1
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Mar 10 11:10:39 2020 -0700
Add construction of domain in Helix participant logic (#876)
Add the construction of domain field in Azure cloud instance information processor. It will read the topology from AzureConstants, and construct the domain based on the format of the topology.
---
.../helix/cloud/azure/AzureCloudInstanceInformationProcessor.java | 7 ++++++-
.../main/java/org/apache/helix/manager/zk/ParticipantManager.java | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 85e2bb5..464ff55 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -142,8 +142,13 @@ public class AzureCloudInstanceInformationProcessor
String vmName = computeNode.path(INSTANCE_NAME).getTextValue();
String platformFaultDomain = computeNode.path(DOMAIN).getTextValue();
String vmssName = computeNode.path(INSTANCE_SET_NAME).getValueAsText();
+ String azureTopology = AzureConstants.AZURE_TOPOLOGY;
+ String[] parts = azureTopology.trim().split("/");
+ //The hostname will be filled in by each participant
+ String domain = parts[0] + "=" + platformFaultDomain + "," + parts[1] + "=";
+
AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
- builder.setInstanceName(vmName).setFaultDomain(platformFaultDomain)
+ builder.setInstanceName(vmName).setFaultDomain(domain)
.setInstanceSetName(vmssName);
azureCloudInstanceInformation = builder.build();
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 7ae62b8..7063c20 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -178,7 +178,7 @@ public class ParticipantManager {
LOG.info(_instanceName + " is auto-registering cluster: " + _clusterName);
CloudInstanceInformation cloudInstanceInformation = getCloudInstanceInformation();
String domain = cloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name());
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()) + _instanceName;
// Disable the verification for now
/*String cloudIdInRemote = cloudInstanceInformation
[helix] 14/18: Change TestInstanceAutoJoin to adapt to cloud
environment (#1265)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ef5722e2943df24a3920eb448a1af6f2b3e7ccb8
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Thu Aug 13 16:10:54 2020 -0700
Change TestInstanceAutoJoin to adapt to cloud environment (#1265)
Change TestInstanceAutoJoin to adapt to cloud environment
---
.../AzureCloudInstanceInformationProcessor.java | 6 +++---
.../paticipant/TestInstanceAutoJoin.java | 25 ++++++++++++++--------
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index f0f75f7..c943664 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -109,7 +109,7 @@ public class AzureCloudInstanceInformationProcessor
CloseableHttpResponse response = _closeableHttpClient.execute(httpGet);
if (response == null || response.getStatusLine().getStatusCode() != 200) {
String errorMsg = String.format(
- "Failed to get an HTTP Response for the request. Response: {}. Status code: {}",
+ "Failed to get an HTTP Response for the request. Response: %s. Status code: %s",
(response == null ? "NULL" : response.getStatusLine().getReasonPhrase()),
response.getStatusLine().getStatusCode());
throw new HelixException(errorMsg);
@@ -119,7 +119,7 @@ public class AzureCloudInstanceInformationProcessor
return responseString;
} catch (IOException e) {
throw new HelixException(
- String.format("Failed to get Azure cloud instance information from url {}", url), e);
+ String.format("Failed to get Azure cloud instance information from url %s", url), e);
}
}
@@ -153,7 +153,7 @@ public class AzureCloudInstanceInformationProcessor
}
} catch (IOException e) {
throw new HelixException(
- String.format("Error in parsing cloud instance information: {}", response, e));
+ String.format("Error in parsing cloud instance information: %s", response, e));
}
return azureCloudInstanceInformation;
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
index 5d5ec46..7f78559 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
@@ -4,6 +4,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.PropertyKey;
+import org.apache.helix.TestHelper;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
import org.apache.helix.integration.manager.MockParticipantManager;
@@ -15,8 +16,6 @@ import org.apache.helix.model.builder.ConfigScopeBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
-import static org.testng.Assert.*;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -83,12 +82,11 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
}
@Test(dependsOnMethods = "testInstanceAutoJoin")
- public void testAutoRegistrationShouldFailWhenWaitingResponse() throws Exception {
+ public void testAutoRegistration() throws Exception {
// Create CloudConfig object and add to config
CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
cloudConfigBuilder.setCloudEnabled(true);
cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
- cloudConfigBuilder.setCloudID("TestID");
CloudConfig cloudConfig = cloudConfigBuilder.build();
HelixManager manager = _participants[0];
@@ -110,14 +108,23 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance3);
autoParticipant.syncStart();
- Assert.assertTrue(null == manager.getHelixDataAccessor()
- .getProperty(accessor.keyBuilder().liveInstance(instance3)));
+ // if the test is run in cloud environment, auto registration will succeed and live instance
+ // will be added, otherwise, auto registration will fail and instance config will not be
+ // populated. An exception will be thrown.
try {
manager.getConfigAccessor().getInstanceConfig(CLUSTER_NAME, instance3);
- fail(
- "Exception should be thrown because the instance cannot be added to the cluster due to the disconnection with Azure endpoint");
+ Assert.assertTrue(TestHelper.verify(() -> {
+ if (null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance3))) {
+ return false;
+ }
+ return true;
+ }, 2000));
} catch (HelixException e) {
-
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance3)));
}
+
+ autoParticipant.syncStop();
}
}
[helix] 09/18: Change the cluster creation logic (#872)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 2ee9d3c0f3086b02bb8ee35b6ec352990d4be8d3
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800
Change the cluster creation logic (#872)
* Change the cluster creation logic
In this commit, minor modifications have been added which:
1- Update topology information in the Cluster Config
if cloud is enabled for Azure environment.
2- Tests have been change accordingly.
3- The deprecated APIs have been changed in the test to
follow new APIs.
---
.../org/apache/helix/cloud/azure/AzureConstants.java | 6 ++++++
.../main/java/org/apache/helix/tools/ClusterSetup.java | 13 ++++++++++++-
.../java/org/apache/helix/tools/TestClusterSetup.java | 14 +++++++++++---
.../apache/helix/rest/server/TestClusterAccessor.java | 16 +++++++++++-----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+ public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+ public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 27d8c88..9663223 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -36,12 +36,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
-import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -171,6 +172,16 @@ public class ClusterSetup {
if (cloudConfig != null) {
_admin.addCloudConfig(clusterName, cloudConfig);
+ // If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
+ if (cloudConfig.isCloudEnabled()
+ && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ configAccessor.updateClusterConfig(clusterName, clusterConfig);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 6a6d48d..1772521 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -33,12 +33,14 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
@@ -354,7 +356,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -420,7 +422,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -504,12 +506,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ // Since it is Azure, topology information should have been populated.
+ ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
// Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
Assert.assertNull(listUrlFromZk);
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 2df28bd..8102db7 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -38,6 +38,7 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterDistributedController;
@@ -590,11 +591,16 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
}
@Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -665,7 +671,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -695,7 +701,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -754,7 +760,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -798,7 +804,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertNull(cloudConfigFromZk);
[helix] 05/18: Add Helix properties factory and class (#653)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit bc3dd83398f9861e5fc4bb0b44888c29c7bbe355
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Jan 21 14:05:50 2020 -0800
Add Helix properties factory and class (#653)
- Add Helix property singleton factory. The factory returns Helix property with default values, and clients may override these values;
- Add Helix manager property. It further holds Helix cloud properties and some other properties specific to the manager, defined in helix-manager.properties file.
- Add Helix cloud property. It holds cloud related properties, which comes from cloud config and client specified file, e.g., for Azure, the file is azure-cloud.properties.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 2 -
.../java/org/apache/helix/HelixCloudProperty.java | 183 +++++++++++++++++++++
.../java/org/apache/helix/HelixManagerFactory.java | 2 +-
.../org/apache/helix/HelixManagerProperty.java | 74 +++++++++
.../org/apache/helix/HelixPropertyFactory.java | 79 +++++++++
.../java/org/apache/helix/SystemPropertyKeys.java | 8 +
.../cloud/azure/AzureCloudInstanceInformation.java | 18 +-
.../helix/manager/zk/ParticipantManager.java | 10 ++
.../apache/helix/manager/zk/ZKHelixManager.java | 17 +-
.../java/org/apache/helix/model/CloudConfig.java | 36 +++-
.../src/main/resources/azure-cloud.properties | 25 +++
.../src/main/resources/helix-manager.properties | 24 +++
12 files changed, 453 insertions(+), 25 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index d61b4ee..f1e3ffa 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -587,9 +587,7 @@ public class ConfigAccessor {
/**
* Get RestConfig of the given cluster.
- *
* @param clusterName The cluster
- *
* @return The instance of {@link RESTConfig}
*/
public RESTConfig getRESTConfig(String clusterName) {
diff --git a/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
new file mode 100644
index 0000000..554c595
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
@@ -0,0 +1,183 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hold helix cloud properties read from CloudConfig and user defined files. Clients may override
+ * the fields from their application.
+ */
+public class HelixCloudProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixCloudProperty.class.getName());
+ private static final String AZURE_CLOUD_PROPERTY_FILE = SystemPropertyKeys.AZURE_CLOUD_PROPERTIES;
+ private static final String CLOUD_INFO_SOURCE = "cloud_info_source";
+ private static final String CLOUD_INFO_PROCESSFOR_NAME = "cloud_info_processor_name";
+ private static final String CLOUD_MAX_RETRY = "cloud_max_retry";
+ private static final String CONNECTION_TIMEOUT_MS = "connection_timeout_ms";
+ private static final String REQUEST_TIMEOUT_MS = "request_timeout_ms";
+
+ // Denote whether the instance is considered as in a cloud environment.
+ private boolean _isCloudEnabled;
+
+ // Unique id of the cloud environment where the instance is in.
+ private String _cloudId;
+
+ // Cloud environment provider, e.g. Azure, AWS, GCP, etc.
+ private String _cloudProvider;
+
+ // The sources where the cloud instance information can be retrieved from.
+ private List<String> _cloudInfoSources;
+
+ // The name of the function that will fetch and parse cloud instance information.
+ private String _cloudInfoProcessorName;
+
+ // Http max retry times when querying the cloud instance information from cloud environment.
+ private int _cloudMaxRetry;
+
+ // Http connection time when querying the cloud instance information from cloud environment.
+ private long _cloudConnectionTimeout;
+
+ // Http request timeout when querying the cloud instance information from cloud environment.
+ private long _cloudRequestTimeout;
+
+ // Other customized properties that may be used.
+ private Properties _customizedCloudProperties = new Properties();
+
+ /**
+ * Initialize Helix Cloud Property based on the provider
+ * @param
+ */
+ public HelixCloudProperty(CloudConfig cloudConfig) {
+ setCloudEndabled(cloudConfig.isCloudEnabled());
+ if (cloudConfig.isCloudEnabled()) {
+ setCloudId(cloudConfig.getCloudID());
+ setCloudProvider(cloudConfig.getCloudProvider());
+ switch (CloudProvider.valueOf(cloudConfig.getCloudProvider())) {
+ case AZURE:
+ Properties azureProperties = new Properties();
+ try {
+ InputStream stream = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(AZURE_CLOUD_PROPERTY_FILE);
+ azureProperties.load(stream);
+ } catch (IOException e) {
+ String errMsg =
+ "failed to open Helix Azure cloud properties file: " + AZURE_CLOUD_PROPERTY_FILE;
+ throw new IllegalArgumentException(errMsg, e);
+ }
+ LOG.info("Successfully loaded Helix Azure cloud properties: {}", azureProperties);
+ setCloudInfoSources(
+ Collections.singletonList(azureProperties.getProperty(CLOUD_INFO_SOURCE)));
+ setCloudInfoProcessorName(azureProperties.getProperty(CLOUD_INFO_PROCESSFOR_NAME));
+ setCloudMaxRetry(Integer.valueOf(azureProperties.getProperty(CLOUD_MAX_RETRY)));
+ setCloudConnectionTimeout(Long.valueOf(azureProperties.getProperty(CONNECTION_TIMEOUT_MS)));
+ setCloudRequestTimeout(Long.valueOf(azureProperties.getProperty(REQUEST_TIMEOUT_MS)));
+ break;
+ case CUSTOMIZED:
+ setCloudInfoSources(cloudConfig.getCloudInfoSources());
+ setCloudInfoProcessorName(cloudConfig.getCloudInfoProcessorName());
+ break;
+ default:
+ throw new HelixException(
+ String.format("Unsupported cloud provider: %s", cloudConfig.getCloudProvider()));
+ }
+ }
+ }
+
+ public boolean getCloudEnabled() {
+ return _isCloudEnabled;
+ }
+
+ public String getCloudId() {
+ return _cloudId;
+ }
+
+ public String getCloudProvider() {
+ return _cloudProvider;
+ }
+
+ public List<String> getCloudInfoSources() {
+ return _cloudInfoSources;
+ }
+
+ public String getCloudInfoProcessorName() {
+ return _cloudInfoProcessorName;
+ }
+
+ public int getCloudMaxRetry() {
+ return _cloudMaxRetry;
+ }
+
+ public long getCloudConnectionTimeout() {
+ return _cloudConnectionTimeout;
+ }
+
+ public long getCloudRequestTimeout() {
+ return _cloudRequestTimeout;
+ }
+
+ public Properties getCustomizedCloudProperties() {
+ return _customizedCloudProperties;
+ }
+
+ public void setCloudEndabled(boolean isCloudEnabled) {
+ _isCloudEnabled = isCloudEnabled;
+ }
+
+ public void setCloudId(String cloudId) {
+ _cloudId = cloudId;
+ }
+
+ public void setCloudProvider(String cloudProvider) {
+ _cloudProvider = cloudProvider;
+ }
+
+ public void setCloudInfoSources(List<String> sources) {
+ _cloudInfoSources = sources;
+ }
+
+ public void setCloudInfoProcessorName(String cloudInfoProcessorName) {
+ _cloudInfoProcessorName = cloudInfoProcessorName;
+ }
+
+ public void setCloudMaxRetry(int cloudMaxRetry) {
+ _cloudMaxRetry = cloudMaxRetry;
+ }
+
+ public void setCloudConnectionTimeout(long cloudConnectionTimeout) {
+ _cloudConnectionTimeout = cloudConnectionTimeout;
+ }
+
+ public void setCloudRequestTimeout(long cloudRequestTimeout) {
+ _cloudRequestTimeout = cloudRequestTimeout;
+ }
+
+ public void setCustomizedCloudProperties(Properties customizedCloudProperties) {
+ _customizedCloudProperties.putAll(customizedCloudProperties);
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java b/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
index 7d8038d..ee53c1d 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
* Obtain one of a set of Helix cluster managers, organized by the backing system.
*/
public final class HelixManagerFactory {
- private static final Logger logger = LoggerFactory.getLogger(HelixManagerFactory.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HelixManagerFactory.class);
/**
* Construct a zk-based cluster manager that enforces all types (PARTICIPANT, CONTROLLER, and
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java b/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
new file mode 100644
index 0000000..2ec26a7
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
@@ -0,0 +1,74 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Properties;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hold Helix manager properties. The manager properties further hold Helix cloud properties
+ * and some other properties specific for the manager.
+ */
+public class HelixManagerProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixManagerProperty.class.getName());
+ private String _version;
+ private long _healthReportLatency;
+ private HelixCloudProperty _helixCloudProperty;
+
+ /**
+ * Initialize Helix manager property with default value
+ * @param helixManagerProperties helix manager related properties input as a map
+ * @param cloudConfig cloudConfig read from Zookeeper
+ */
+ public HelixManagerProperty(Properties helixManagerProperties, CloudConfig cloudConfig) {
+ _helixCloudProperty = new HelixCloudProperty(cloudConfig);
+ setVersion(helixManagerProperties.getProperty(SystemPropertyKeys.HELIX_MANAGER_VERSION));
+ setHealthReportLatency(
+ helixManagerProperties.getProperty(SystemPropertyKeys.PARTICIPANT_HEALTH_REPORT_LATENCY));
+ }
+
+ public HelixCloudProperty getHelixCloudProperty() {
+ return _helixCloudProperty;
+ }
+
+ public String getVersion() {
+ return _version;
+ }
+
+ public long getHealthReportLatency() {
+ return _healthReportLatency;
+ }
+
+ public void setHelixCloudProperty(HelixCloudProperty helixCloudProperty) {
+ _helixCloudProperty = helixCloudProperty;
+ }
+
+ public void setVersion(String version) {
+ _version = version;
+ }
+
+ public void setHealthReportLatency(String latency) {
+ _healthReportLatency = Long.valueOf(latency);
+ }
+
+ // TODO: migrate all other participant related properties to this file.
+}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
new file mode 100644
index 0000000..fa394a2
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
@@ -0,0 +1,79 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Singleton factory that builds different types of Helix property, e.g. Helix manager property.
+ */
+public final class HelixPropertyFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixPropertyFactory.class);
+ private static final String HELIX_PARTICIPANT_PROPERTY_FILE =
+ SystemPropertyKeys.HELIX_MANAGER_PROPERTIES;
+
+ private static class SingletonHelper {
+ private static final HelixPropertyFactory INSTANCE = new HelixPropertyFactory();
+ }
+
+ public static HelixPropertyFactory getInstance() {
+ return SingletonHelper.INSTANCE;
+ }
+
+ /**
+ * Retrieve Helix manager property. It returns the property object with default values.
+ * Clients may override these values.
+ */
+ public HelixManagerProperty getHelixManagerProperty(String zkAddress, String clusterName) {
+ ConfigAccessor configAccessor = new ConfigAccessor(zkAddress);
+ CloudConfig cloudConfig;
+ // The try-catch logic is for backward compatibility reason only. Even if the cluster is not set
+ // up yet, constructing a new ZKHelixManager should not throw an exception
+ try {
+ cloudConfig =
+ configAccessor.getCloudConfig(clusterName) == null ? buildEmptyCloudConfig(clusterName)
+ : configAccessor.getCloudConfig(clusterName);
+ } catch (HelixException e) {
+ cloudConfig = buildEmptyCloudConfig(clusterName);
+ }
+ Properties properties = new Properties();
+ try {
+ InputStream stream = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(HELIX_PARTICIPANT_PROPERTY_FILE);
+ properties.load(stream);
+ } catch (IOException e) {
+ String errMsg = String.format("failed to open Helix participant properties file: %s",
+ HELIX_PARTICIPANT_PROPERTY_FILE);
+ throw new IllegalArgumentException(errMsg, e);
+ }
+ LOG.info("HelixPropertyFactory successfully loaded helix participant properties: {}",
+ properties);
+ return new HelixManagerProperty(properties, cloudConfig);
+ }
+
+ public static CloudConfig buildEmptyCloudConfig(String clusterName) {
+ return new CloudConfig.Builder().setCloudEnabled(false).build();
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java b/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
index f967b71..93ca7b4 100644
--- a/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
+++ b/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
@@ -1,6 +1,14 @@
package org.apache.helix;
public class SystemPropertyKeys {
+ // Used to compose default values in HelixManagerProperty
+ public static final String HELIX_MANAGER_PROPERTIES = "helix-manager.properties";
+
+ public static final String HELIX_MANAGER_VERSION = "clustermanager.version";
+
+ // Used to compose default values in HelixCloudProperty when cloud provider is Azure
+ public static final String AZURE_CLOUD_PROPERTIES = "azure-cloud.properties";
+
// Task Driver
public static final String TASK_CONFIG_LIMITATION = "helixTask.configsLimitation";
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
index 511f3e3..f7fd657 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -48,24 +48,18 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
}
- /**
- * Default constructor
- */
- public Builder() {
- }
-
- public Builder setInstanceName(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), v);
+ public Builder setInstanceName(String name) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), name);
return this;
}
- public Builder setFaultDomain(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), v);
+ public Builder setFaultDomain(String faultDomain) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), faultDomain);
return this;
}
- public Builder setInstanceSetName(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), v);
+ public Builder setInstanceSetName(String instanceSetName) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), instanceSetName);
return this;
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 2fc0cf4..f25f29e 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -33,6 +33,7 @@ import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
import org.apache.helix.PreConnectCallback;
@@ -76,9 +77,17 @@ public class ParticipantManager {
final StateMachineEngine _stateMachineEngine;
final LiveInstanceInfoProvider _liveInstanceInfoProvider;
final List<PreConnectCallback> _preConnectCallbacks;
+ final HelixManagerProperty _helixManagerProperty;
+ @Deprecated
public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout,
LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks) {
+ this(manager, zkclient, sessionTimeout, liveInstanceInfoProvider, preConnectCallbacks, null);
+ }
+
+ public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout,
+ LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks,
+ HelixManagerProperty helixManagerProperty) {
_zkclient = zkclient;
_manager = manager;
_clusterName = manager.getClusterName();
@@ -94,6 +103,7 @@ public class ParticipantManager {
_stateMachineEngine = manager.getStateMachineEngine();
_liveInstanceInfoProvider = liveInstanceInfoProvider;
_preConnectCallbacks = preConnectCallbacks;
+ _helixManagerProperty = helixManagerProperty;
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index 31d45c0..60f57c0 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -41,6 +41,8 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerProperties;
+import org.apache.helix.HelixManagerProperty;
+import org.apache.helix.HelixPropertyFactory;
import org.apache.helix.HelixTimerTask;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
@@ -105,6 +107,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
private final List<PreConnectCallback> _preConnectCallbacks;
protected final List<CallbackHandler> _handlers;
private final HelixManagerProperties _properties;
+ private final HelixManagerProperty _helixManagerProperty;
private final HelixManagerStateListener _stateListener;
/**
@@ -201,9 +204,16 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
public ZKHelixManager(String clusterName, String instanceName, InstanceType instanceType,
String zkAddress, HelixManagerStateListener stateListener) {
+ this(clusterName, instanceName, instanceType, zkAddress, stateListener,
+ HelixPropertyFactory.getInstance().getHelixManagerProperty(zkAddress, clusterName));
+ }
- LOG.info(
- "Create a zk-based cluster manager. zkSvr: " + zkAddress + ", clusterName: " + clusterName + ", instanceName: " + instanceName + ", type: " + instanceType);
+ public ZKHelixManager(String clusterName, String instanceName, InstanceType instanceType,
+ String zkAddress, HelixManagerStateListener stateListener,
+ HelixManagerProperty helixManagerProperty) {
+
+ LOG.info("Create a zk-based cluster manager. zkSvr: " + zkAddress + ", clusterName: "
+ + clusterName + ", instanceName: " + instanceName + ", type: " + instanceType);
_zkAddress = zkAddress;
_clusterName = clusterName;
@@ -243,6 +253,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
_stateListener = stateListener;
+ _helixManagerProperty = helixManagerProperty;
/**
* use system property if available
@@ -1167,7 +1178,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
_participantManager =
new ParticipantManager(this, _zkclient, _sessionTimeout, _liveInstanceInfoProvider,
- _preConnectCallbacks);
+ _preConnectCallbacks, _helixManagerProperty);
_participantManager.handleNewSession();
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index f6c279c..79c7330 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -71,6 +71,26 @@ public class CloudConfig extends HelixProperty {
_record.setMapFields(record.getMapFields());
}
+ /**
+ * Instantiate the config using each field individually.
+ * Users should use CloudConfig.Builder to create CloudConfig.
+ * @param cluster
+ * @param enabled
+ * @param cloudID
+ */
+ public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
+ List<String> cloudInfoSource, String cloudProcessorName) {
+ super(cluster);
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ if (enabled == true) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
+ }
+ }
+ }
/**
* Enable/Disable the CLOUD_ENABLED field.
@@ -226,14 +246,16 @@ public class CloudConfig extends HelixProperty {
}
private void validate() {
- if (this.getCloudProvider() == null) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
- } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
- if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
- || this.getCloudInfoSources().size() == 0) {
+ if (this.getCloudEnabled()) {
+ if (this.getCloudProvider() == null) {
throw new HelixException(
- "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
+ if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
+ || this.getCloudInfoSources().size() == 0) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ }
}
}
}
diff --git a/helix-core/src/main/resources/azure-cloud.properties b/helix-core/src/main/resources/azure-cloud.properties
new file mode 100644
index 0000000..b5321fc
--- /dev/null
+++ b/helix-core/src/main/resources/azure-cloud.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This is the globally fixed value for Azure Metadata Instance Service endpoint
+cloud_info_source=http://169.254.169.254/metadata/instance?api-version=2019-06-04
+cloud_info_processor_name=AzureCloudInstanceInformationProcessor
+cloud_max_retry=5
+connection_timeout_ms=5000
+request_timeout_ms=5000
\ No newline at end of file
diff --git a/helix-core/src/main/resources/helix-manager.properties b/helix-core/src/main/resources/helix-manager.properties
new file mode 100644
index 0000000..eafcb3a
--- /dev/null
+++ b/helix-core/src/main/resources/helix-manager.properties
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+clustermanager.version=${project.version}
+
+minimum_supported_version.participant=0.4
+
+helixmanager.participantHealthReport.reportLatency=60000
\ No newline at end of file
[helix] 18/18: Return "name" field as VM name in Azure environment
(#1340)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 681fdbb20c4c4fc220c6d5c5d9621801e84c9802
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Wed Sep 2 23:02:16 2020 -0700
Return "name" field as VM name in Azure environment (#1340)
* Return name field as VM name in Azure envrionment
* fix comment
* fix test
---
.../src/main/java/org/apache/helix/HelixCloudProperty.java | 8 ++++----
.../azure/AzureCloudInstanceInformationProcessor.java | 14 +++++++-------
.../org/apache/helix/manager/zk/ParticipantManager.java | 4 ++--
.../java/org/apache/helix/util/InstanceValidationUtil.java | 14 +++++++++++++-
.../cloud/TestAzureCloudInstanceInformationProcessor.java | 2 +-
5 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
index 554c595..6cbc2e1 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
@@ -37,7 +37,7 @@ public class HelixCloudProperty {
private static final Logger LOG = LoggerFactory.getLogger(HelixCloudProperty.class.getName());
private static final String AZURE_CLOUD_PROPERTY_FILE = SystemPropertyKeys.AZURE_CLOUD_PROPERTIES;
private static final String CLOUD_INFO_SOURCE = "cloud_info_source";
- private static final String CLOUD_INFO_PROCESSFOR_NAME = "cloud_info_processor_name";
+ private static final String CLOUD_INFO_PROCESSOR_NAME = "cloud_info_processor_name";
private static final String CLOUD_MAX_RETRY = "cloud_max_retry";
private static final String CONNECTION_TIMEOUT_MS = "connection_timeout_ms";
private static final String REQUEST_TIMEOUT_MS = "request_timeout_ms";
@@ -74,7 +74,7 @@ public class HelixCloudProperty {
* @param
*/
public HelixCloudProperty(CloudConfig cloudConfig) {
- setCloudEndabled(cloudConfig.isCloudEnabled());
+ setCloudEnabled(cloudConfig.isCloudEnabled());
if (cloudConfig.isCloudEnabled()) {
setCloudId(cloudConfig.getCloudID());
setCloudProvider(cloudConfig.getCloudProvider());
@@ -93,7 +93,7 @@ public class HelixCloudProperty {
LOG.info("Successfully loaded Helix Azure cloud properties: {}", azureProperties);
setCloudInfoSources(
Collections.singletonList(azureProperties.getProperty(CLOUD_INFO_SOURCE)));
- setCloudInfoProcessorName(azureProperties.getProperty(CLOUD_INFO_PROCESSFOR_NAME));
+ setCloudInfoProcessorName(azureProperties.getProperty(CLOUD_INFO_PROCESSOR_NAME));
setCloudMaxRetry(Integer.valueOf(azureProperties.getProperty(CLOUD_MAX_RETRY)));
setCloudConnectionTimeout(Long.valueOf(azureProperties.getProperty(CONNECTION_TIMEOUT_MS)));
setCloudRequestTimeout(Long.valueOf(azureProperties.getProperty(REQUEST_TIMEOUT_MS)));
@@ -145,7 +145,7 @@ public class HelixCloudProperty {
return _customizedCloudProperties;
}
- public void setCloudEndabled(boolean isCloudEnabled) {
+ public void setCloudEnabled(boolean isCloudEnabled) {
_isCloudEnabled = isCloudEnabled;
}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index c943664..b11e943 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -48,15 +48,15 @@ public class AzureCloudInstanceInformationProcessor
LoggerFactory.getLogger(AzureCloudInstanceInformationProcessor.class);
private final CloseableHttpClient _closeableHttpClient;
private final HelixCloudProperty _helixCloudProperty;
- private final String COMPUTE = "compute";
- private final String INSTANCE_NAME = "vmId";
- private final String DOMAIN = "platformFaultDomain";
- private final String INSTANCE_SET_NAME = "vmScaleSetName";
+ private static final String COMPUTE = "compute";
+ private static final String INSTANCE_NAME = "name";
+ private static final String DOMAIN = "platformFaultDomain";
+ private static final String INSTANCE_SET_NAME = "vmScaleSetName";
public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty) {
_helixCloudProperty = helixCloudProperty;
- RequestConfig requestConifg = RequestConfig.custom()
+ RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout((int) helixCloudProperty.getCloudRequestTimeout())
.setConnectTimeout((int) helixCloudProperty.getCloudConnectionTimeout()).build();
@@ -69,7 +69,7 @@ public class AzureCloudInstanceInformationProcessor
};
//TODO: we should regularize the way how httpClient should be used throughout Helix. e.g. Helix-rest could also use in the same way
- _closeableHttpClient = HttpClients.custom().setDefaultRequestConfig(requestConifg)
+ _closeableHttpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig)
.setRetryHandler(httpRequestRetryHandler).build();
}
@@ -153,7 +153,7 @@ public class AzureCloudInstanceInformationProcessor
}
} catch (IOException e) {
throw new HelixException(
- String.format("Error in parsing cloud instance information: %s", response, e));
+ String.format("Error in parsing cloud instance information: %s", response), e);
}
return azureCloudInstanceInformation;
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 7063c20..bdf820f 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -143,7 +143,7 @@ public class ParticipantManager {
boolean autoJoin = false;
boolean autoRegistration = false;
- // Read "allowParticipantAutoJoin" flag to see if an instance can auto join to the cluster
+ // Read "allowParticipantAutoJoin" field to see if an instance can auto join to the cluster
try {
HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
.forCluster(_manager.getClusterName()).build();
@@ -160,7 +160,7 @@ public class ParticipantManager {
try {
autoRegistration =
Boolean.valueOf(_helixManagerProperty.getHelixCloudProperty().getCloudEnabled());
- LOG.info("instance: " + _instanceName + " auto-register " + _clusterName + " is "
+ LOG.info("instance: " + _instanceName + " auto-registering " + _clusterName + " is "
+ autoRegistration);
} catch (Exception e) {
LOG.info("auto registration is false for cluster" + _clusterName);
diff --git a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
index 30b733d..b80487d 100644
--- a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java
@@ -155,7 +155,19 @@ public class InstanceValidationUtil {
*/
public static boolean hasValidConfig(HelixDataAccessor dataAccessor, String clusterId,
String instanceName) {
- PropertyKey propertyKey = dataAccessor.keyBuilder().instanceConfig(instanceName);
+ PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
+ ClusterConfig clusterConfig = dataAccessor.getProperty(keyBuilder.clusterConfig());
+ if (clusterConfig == null) {
+ _logger.error("Cluster config is missing in cluster " + clusterId);
+ return false;
+ }
+ if (!clusterConfig.isPersistIntermediateAssignment()) {
+ _logger.error(
+ "Cluster config %s is not turned on, which is required for instance stability check.",
+ ClusterConfig.ClusterConfigProperty.PERSIST_INTERMEDIATE_ASSIGNMENT.toString());
+ return false;
+ }
+ PropertyKey propertyKey = keyBuilder.instanceConfig(instanceName);
InstanceConfig instanceConfig = dataAccessor.getProperty(propertyKey);
return instanceConfig != null && instanceConfig.isValid();
}
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
index 350c9a9..4bad9d7 100644
--- a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -64,6 +64,6 @@ public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
Assert.assertEquals(
azureCloudInstanceInformation
.get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
- "d2b921cc-c16c-41f7-a86d-a445eac6ec26");
+ "test-helix_1");
}
}
[helix] 01/18: Add CloudConfig code
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 6800dc352ae36aa07c0b320f2ce861223b05697d
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Nov 11 11:28:32 2019 -0800
Add CloudConfig code
In order to move toward supporting cloud environments and autoregisterations,
we need to add CloudConfig to Zookeeper.
The code regarding CloudConfig is added.
A new test has been added to check the correctness of the code.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 23 ++
.../main/java/org/apache/helix/PropertyKey.java | 11 +
.../helix/cloud/constants/CloudProvider.java | 25 ++
.../java/org/apache/helix/model/CloudConfig.java | 290 +++++++++++++++++++++
.../org/apache/helix/model/HelixConfigScope.java | 6 +-
.../model/builder/HelixConfigScopeBuilder.java | 3 +
.../apache/helix/model/cloud/TestCloudConfig.java | 204 +++++++++++++++
7 files changed, 561 insertions(+), 1 deletion(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 15ea4a0..9bd84d1 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
@@ -563,6 +564,28 @@ public class ConfigAccessor {
}
/**
+ * Get CloudConfig of the given cluster.
+ * @param clusterName
+ * @return The instance of {@link CloudConfig}
+ */
+ public CloudConfig getCloudConfig(String clusterName) {
+ if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
+ throw new HelixException(
+ String.format("Failed to get config. cluster: %s is not setup.", clusterName));
+ }
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ ZNRecord record = getConfigZnRecord(scope);
+
+ if (record == null) {
+ LOG.warn("No cloud config found at {}.", scope.getZkPath());
+ return null;
+ }
+
+ return new CloudConfig(record);
+ }
+
+ /**
* Get RestConfig of the given cluster.
*
* @param clusterName The cluster
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 369e48e..533cfc1 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -24,6 +24,7 @@ import static org.apache.helix.PropertyType.*;
import java.util.Arrays;
import java.util.Objects;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ControllerHistory;
@@ -214,6 +215,16 @@ public class PropertyKey {
_clusterName, ConfigScopeProperty.CLUSTER.toString(), _clusterName);
}
+
+ /**
+ * Get a property key associated with this Cloud configuration
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey cloudConfig() {
+ return new PropertyKey(CONFIGS, ConfigScopeProperty.CLOUD, CloudConfig.class,
+ _clusterName, ConfigScopeProperty.CLOUD.name(), _clusterName);
+ }
+
/**
* Get a property key associated with {@link InstanceConfig}
* @return {@link PropertyKey}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/constants/CloudProvider.java b/helix-core/src/main/java/org/apache/helix/cloud/constants/CloudProvider.java
new file mode 100644
index 0000000..1cb836f
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/constants/CloudProvider.java
@@ -0,0 +1,25 @@
+package org.apache.helix.cloud.constants;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public enum CloudProvider {
+ AZURE,
+ CUSTOMIZED
+}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
new file mode 100644
index 0000000..c8ab6eb
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -0,0 +1,290 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.constants.CloudProvider;
+
+/**
+ * Cloud configurations
+ */
+public class CloudConfig extends HelixProperty {
+ /**
+ * Configurable characteristics of a cloud.
+ * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
+ * CloudConfig.
+ */
+ public enum CloudConfigProperty {
+ CLOUD_ENABLED, // determine whether the cluster is inside cloud environment.
+ CLOUD_PROVIDER, // the environment the cluster is in, e.g. Azure, AWS, or Customized
+ CLOUD_ID, // the cloud Id that belongs to this cluster.
+
+ // If user uses Helix supported default provider, the below entries will not be shown in
+ // CloudConfig.
+ CLOUD_INFO_SOURCE, // the source for retrieving the cloud information.
+ CLOUD_INFO_PROCESSOR_NAME // the name of the function that processes the fetching and parsing of
+ // cloud information.
+ }
+
+ /* Default values */
+ private static final boolean DEFAULT_CLOUD_ENABLED = false;
+
+ /**
+ * Instantiate the CloudConfig for the cloud
+ * @param cluster
+ */
+ public CloudConfig(String cluster) {
+ super(cluster);
+ }
+
+ /**
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a cloud configuration
+ */
+ public CloudConfig(ZNRecord record) {
+ super(record);
+ }
+
+ /**
+ * Instantiate the config using each field individually.
+ * Users should use CloudConfig.Builder to create CloudConfig.
+ * @param cluster
+ * @param enabled
+ * @param cloudID
+ */
+ public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
+ List<String> cloudInfoSource, String cloudProcessorName) {
+ super(cluster);
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
+ _record
+ .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
+ }
+ }
+
+ /**
+ * Enable/Disable the CLOUD_ENABLED field.
+ * @param enabled
+ */
+ public void setCloudEnabled(boolean enabled) {
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ }
+
+ /**
+ * Whether CLOUD_ENABLED field is enabled or not.
+ * @return
+ */
+ public boolean isCloudEnabled() {
+ return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), false);
+ }
+
+ /**
+ * Set the cloudID field.
+ * @param cloudID
+ */
+ public void setCloudID(String cloudID) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ }
+
+ /**
+ * Get the CloudID field.
+ * @return CloudID
+ */
+ public String getCloudID() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name());
+ }
+
+ /**
+ * Set the CLOUD_INFO_SOURCE field.
+ * @param cloudInfoSources
+ */
+ public void setCloudInfoSource(List<String> cloudInfoSources) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources);
+ }
+
+ /**
+ * Get the CLOUD_INFO_SOURCE field.
+ * @return CLOUD_INFO_SOURCE field.
+ */
+ public List<String> getCloudInfoSources() {
+ return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
+ }
+
+ /**
+ * Set the CLOUD_INFO_PROCESSOR_NAME field.
+ * @param cloudInfoProcessorName
+ */
+ public void setCloudInfoFProcessorName(String cloudInfoProcessorName) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ cloudInfoProcessorName);
+ }
+
+ /**
+ * Get the CLOUD_INFO_PROCESSOR_NAME field.
+ * @return CLOUD_INFO_PROCESSOR_NAME field.
+ */
+ public String getCloudInfoProcessorName() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name());
+ }
+
+ /**
+ * Set the CLOUD_PROVIDER field.
+ * @param cloudProvider
+ */
+ public void setCloudProvider(CloudProvider cloudProvider) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ }
+
+ /**
+ * Get the CLOUD_PROVIDER field.
+ * @return CLOUD_PROVIDER field.
+ */
+ public String getCloudProvider() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
+ }
+
+ public static class Builder {
+ private String _clusterName = null;
+ private CloudProvider _cloudProvider;
+ private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED;
+ private String _cloudID;
+ private List<String> _cloudInfoSources;
+ private String _cloudInfoProcessorName;
+
+ public CloudConfig build() {
+ validate();
+ return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID,
+ _cloudInfoSources, _cloudInfoProcessorName);
+ }
+
+ /**
+ * Default constructor
+ */
+ public Builder() {
+ }
+
+ /**
+ * Constructor with Cluster Name as input
+ * @param clusterName
+ */
+ public Builder(String clusterName) {
+ _clusterName = clusterName;
+ }
+
+ /**
+ * Constructor with CloudConfig as input
+ * @param cloudConfig
+ */
+ public Builder(CloudConfig cloudConfig) {
+ _cloudEnabled = cloudConfig.isCloudEnabled();
+ _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider());
+ _cloudID = cloudConfig.getCloudID();
+ _cloudInfoSources = cloudConfig.getCloudInfoSources();
+ _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName();
+ }
+
+ public Builder setClusterName(String v) {
+ _clusterName = v;
+ return this;
+ }
+
+ public Builder setCloudEnabled(boolean isEnabled) {
+ _cloudEnabled = isEnabled;
+ return this;
+ }
+
+ public Builder setCloudProvider(CloudProvider cloudProvider) {
+ _cloudProvider = cloudProvider;
+ return this;
+ }
+
+ public Builder setCloudID(String v) {
+ _cloudID = v;
+ return this;
+ }
+
+ public Builder setCloudInfoSources(List<String> v) {
+ _cloudInfoSources = v;
+ return this;
+ }
+
+ public Builder addCloudInfoSource(String v) {
+ if (_cloudInfoSources == null) {
+ _cloudInfoSources = new ArrayList<String>();
+ }
+ _cloudInfoSources.add(v);
+ return this;
+ }
+
+ public Builder setCloudInfoProcessorName(String v) {
+ _cloudInfoProcessorName = v;
+ return this;
+ }
+
+ public String getClusterName() {
+ return _clusterName;
+ }
+
+ public CloudProvider getCloudProvider() {
+ return _cloudProvider;
+ }
+
+ public boolean getCloudEnabled() {
+ return _cloudEnabled;
+ }
+
+ public String getCloudID() {
+ return _cloudID;
+ }
+
+ public List<String> getCloudInfoSources() {
+ return _cloudInfoSources;
+ }
+
+ public String getCloudInfoProcessorName() {
+ return _cloudInfoProcessorName;
+ }
+
+ private void validate() {
+ if (_cloudEnabled) {
+ if (_cloudID == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The CloudID is missing from the config.");
+ }
+ if (_cloudProvider == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (_cloudProvider == CloudProvider.CUSTOMIZED) {
+ if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
index 7c5c91d..8d814c5 100644
--- a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
+++ b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
@@ -36,7 +36,8 @@ public class HelixConfigScope {
RESOURCE(2, 0),
PARTITION(2, 1),
CONSTRAINT(2, 0),
- REST(2, 0);
+ REST(2, 0),
+ CLOUD(2, 0);
final int _zkPathArgNum;
final int _mapKeyArgNum;
@@ -78,12 +79,15 @@ public class HelixConfigScope {
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}");
template.addEntry(ConfigScopeProperty.PARTITION, 2,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}");
+ template.addEntry(ConfigScopeProperty.CLOUD, 2,
+ "/{clusterName}/CONFIGS/CLOUD/{clusterName}");
// get children
template.addEntry(ConfigScopeProperty.CLUSTER, 1, "/{clusterName}/CONFIGS/CLUSTER");
template.addEntry(ConfigScopeProperty.PARTICIPANT, 1, "/{clusterName}/CONFIGS/PARTICIPANT");
template.addEntry(ConfigScopeProperty.RESOURCE, 1, "/{clusterName}/CONFIGS/RESOURCE");
template.addEntry(ConfigScopeProperty.REST, 2, "/{clusterName}/CONFIGS/REST/{clusterName}");
+ template.addEntry(ConfigScopeProperty.CLOUD, 1, "/{clusterName}/CONFIGS/CLOUD");
}
final ConfigScopeProperty _type;
diff --git a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
index b1a7cf6..78ed074 100644
--- a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
@@ -126,6 +126,9 @@ public class HelixConfigScopeBuilder {
case REST:
scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
break;
+ case CLOUD:
+ scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
+ break;
default:
break;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
new file mode 100644
index 0000000..ee83011
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
@@ -0,0 +1,204 @@
+package org.apache.helix.model.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.PropertyKey.Builder;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CloudConfig;
+import org.apache.helix.cloud.constants.CloudProvider;
+import java.util.List;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestCloudConfig extends ZkUnitTestBase {
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testCloudConfigNonExistentCluster() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ // Read CloudConfig from Zookeeper and get exception since cluster in not setup yet
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ }
+
+ @Test(dependsOnMethods = "testCloudConfigNonExistentCluster")
+ public void testCloudConfigNull() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ // Read CloudConfig from Zookeeper
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ // since CloudConfig is not written to ZooKeeper, the output should be null
+ Assert.assertNull(cloudConfigFromZk);
+ }
+
+ @Test(dependsOnMethods = "testCloudConfigNull")
+ public void testCloudConfig() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+
+ // Create dummy CloudConfig object
+ CloudConfig cloudConfig = new CloudConfig(clusterName);
+ cloudConfig.setCloudEnabled(true);
+ cloudConfig.setCloudProvider(CloudProvider.AZURE);
+ cloudConfig.setCloudID("TestID");
+ List<String> infoURL = new ArrayList<String>();
+ infoURL.add("TestURL");
+ cloudConfig.setCloudInfoSource(infoURL);
+ cloudConfig.setCloudInfoFProcessorName("TestProcessor");
+
+ // Write the CloudConfig to Zookeeper
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1);
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testUnverifiedCloudConfigBuilder() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ // Verify will fail because cloudID has net been defined.
+ CloudConfig cloudConfig = builder.build();
+ }
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testUnverifiedCloudConfigBuilderEmptySources() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudID("TestID");
+ List<String> emptyList = new ArrayList<String>();
+ builder.setCloudInfoSources(emptyList);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+ }
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testUnverifiedCloudConfigBuilderWithoutProcessor() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudID("TestID");
+ List<String> testList = new ArrayList<String>();
+ builder.setCloudInfoSources(testList);
+ builder.addCloudInfoSource("TestURL");
+ CloudConfig cloudConfig = builder.build();
+ }
+
+ @Test(dependsOnMethods = "testCloudConfig")
+ public void testCloudConfigBuilder() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL0");
+ builder.addCloudInfoSource("TestURL1");
+ builder.setCloudInfoProcessorName("TestProcessor");
+
+ // Check builder getter methods
+ Assert.assertTrue(builder.getCloudEnabled());
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED);
+ Assert.assertEquals(builder.getClusterName(), clusterName);
+ Assert.assertEquals(builder.getCloudID(), "TestID");
+ List<String> listUrlFromBuilder = builder.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromBuilder.size(), 2);
+ Assert.assertEquals(listUrlFromBuilder.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromBuilder.get(1), "TestURL1");
+ Assert.assertEquals(builder.getCloudInfoProcessorName(), "TestProcessor");
+
+ CloudConfig cloudConfig = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromZk.get(1), "TestURL1");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+ @Test(dependsOnMethods = "testCloudConfigBuilder")
+ public void testCloudConfigBuilderAzureProvider() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.AZURE);
+ builder.setCloudID("TestID");
+ builder.setCloudInfoProcessorName("TestProcessor");
+
+ // Check builder getter methods
+ Assert.assertTrue(builder.getCloudEnabled());
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE);
+
+ CloudConfig cloudConfig = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null.
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ }
+}
[helix] 11/18: Change the REST call for delete CloudConfig (#882)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 90c9e2583fbbb03ff749410b32009959d2c67ec0
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 13 13:18:07 2020 -0700
Change the REST call for delete CloudConfig (#882)
Change the delete CloudConfig
Use DELETE instead of POST for deletion of the CloudConfig in REST.
---
.../server/resources/helix/ClusterAccessor.java | 12 ++++++---
.../helix/rest/server/TestClusterAccessor.java | 31 +++++++++++++++-------
2 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index f3175e4..4f5bb20 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -586,6 +586,14 @@ public class ClusterAccessor extends AbstractHelixResource {
return notFound();
}
+ @DELETE
+ @Path("{clusterId}/cloudconfig")
+ public Response deleteCloudConfig(@PathParam("clusterId") String clusterId) {
+ HelixAdmin admin = getHelixAdmin();
+ admin.removeCloudConfig(clusterId);
+ return OK();
+ }
+
@POST
@Path("{clusterId}/cloudconfig")
public Response updateCloudConfig(@PathParam("clusterId") String clusterId,
@@ -619,10 +627,6 @@ public class ClusterAccessor extends AbstractHelixResource {
}
try {
switch (command) {
- case delete: {
- admin.removeCloudConfig(clusterId);
- }
- break;
case update: {
try {
CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 8102db7..5bb29ce 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -793,19 +793,32 @@ public class TestClusterAccessor extends AbstractTestClass {
@Test(dependsOnMethods = "testAddCloudConfig")
public void testDeleteCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
- _gSetupTool.addCluster("TestCloud", true);
- String urlBase = "clusters/TestCloud/cloudconfig/";
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
- Map<String, String> map1 = new HashMap<>();
- map1.put("command", AbstractResource.Command.delete.name());
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+ // Read CloudConfig from Zookeeper and make sure it has been created
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNotNull(cloudConfigFromZk);
+ String urlBase = "clusters/" + clusterName + "/cloudconfig/";
- ZNRecord record = new ZNRecord("TestCloud");
- post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
- Response.Status.OK.getStatusCode());
+ delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ _configAccessor = new ConfigAccessor(ZK_ADDR);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk);
System.out.println("End test :" + TestHelper.getTestMethodName());
[helix] 16/18: Add TrieClusterTopology for retrieving hierarchical
topology (#1307)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit bdcb37a1947b4929647a0b8dd0e3d16640bc9d9f
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Thu Sep 24 11:38:51 2020 -0700
Add TrieClusterTopology for retrieving hierarchical topology (#1307)
Add TrieNode class to define a node in the trie
Add ClusterTrie class to handle the construction, validation and retrieval of nodes/paths in the trie.
Add ClusterTopology class to provide different APIs for users to retrieve cluster topology information.
Add APIs in HelixAdmin to retrieve ClusterTopology of a specific cluster.
---
.../src/main/java/org/apache/helix/HelixAdmin.java | 7 +
.../apache/helix/api/topology/ClusterTopology.java | 192 +++++++++++++++++
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 20 ++
.../java/org/apache/helix/model/ClusterTrie.java | 227 +++++++++++++++++++++
.../main/java/org/apache/helix/model/TrieNode.java | 56 +++++
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 76 ++++++-
.../java/org/apache/helix/mock/MockHelixAdmin.java | 6 +
.../org/apache/helix/model/TestClusterTrie.java | 141 +++++++++++++
8 files changed, 724 insertions(+), 1 deletion(-)
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index bb5f3bf..42bcb24 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -22,6 +22,7 @@ package org.apache.helix;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -394,6 +395,12 @@ public interface HelixAdmin {
void removeCloudConfig(String clusterName);
/**
+ * Get the topology of a specific cluster
+ * @param clusterName
+ */
+ ClusterTopology getClusterTopology(String clusterName);
+
+ /**
* Get a list of state model definitions in a cluster
* @param clusterName
* @return
diff --git a/helix-core/src/main/java/org/apache/helix/api/topology/ClusterTopology.java b/helix-core/src/main/java/org/apache/helix/api/topology/ClusterTopology.java
new file mode 100644
index 0000000..72bc594
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/api/topology/ClusterTopology.java
@@ -0,0 +1,192 @@
+package org.apache.helix.api.topology;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.ClusterTrie;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.TrieNode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.helix.model.ClusterTrie.CONNECTOR;
+import static org.apache.helix.model.ClusterTrie.DELIMITER;
+
+
+public class ClusterTopology {
+ private static Logger logger = LoggerFactory.getLogger(ClusterTopology.class);
+
+ private final ClusterTrie _trieClusterTopology;
+
+ public ClusterTopology(final List<String> liveNodes,
+ final Map<String, InstanceConfig> instanceConfigMap, final ClusterConfig clusterConfig) {
+ _trieClusterTopology = new ClusterTrie(liveNodes, instanceConfigMap, clusterConfig);
+ }
+
+ /**
+ * Return the whole topology of a cluster as a map. The key of the map is the first level of
+ * domain, and the value is a list of string that represents the path to each end node in that
+ * domain. E.g., assume the topology is defined as /group/zone/rack/host, the result may be {
+ * ["/group:0": {"/zone:0/rack:0/host:0", "/zone:1/rack:1/host:1"}], ["/group:1": {"/zone:1
+ * /rack:1/host:1", "/zone:1/rack:1/host:2"}]}
+ */
+ public Map<String, List<String>> getTopologyMap() {
+ return getTopologyUnderDomain(Collections.emptyMap());
+ }
+
+ /**
+ * Return all the instances under fault zone type. The key of the returned map is each fault
+ * zone name, and the value is a list of string that represents the path to each end node in
+ * that fault zone.
+ * @return , e.g. if the fault zone is "zone", it may return {["/group:0/zone:0": {"rack:0/host
+ * :0", "rack:1/host:1"}, ["/group:0/zone:1": {"/rack:0:host:2", "/rack:1/host:3"}]}
+ */
+ public Map<String, List<String>> getFaultZoneMap() {
+ String faultZone = _trieClusterTopology.getFaultZoneType();
+ if (faultZone == null) {
+ throw new IllegalArgumentException("The fault zone in cluster config is not defined");
+ }
+ return getTopologyUnderDomainType(faultZone);
+ }
+
+ /**
+ * Return the instances whose domain field is not valid
+ */
+ public List<String> getInvalidInstances() {
+ return _trieClusterTopology.getInvalidInstances();
+ }
+
+ /**
+ * Return the topology under a certain domain as a map. The key of the returned map is the next
+ * level domain, and the value is a list of string that represents the path to each end node in
+ * that domain.
+ * @param domainMap A map defining the domain name and its value, e.g. {["group": "1"], ["zone",
+ * "2"]}
+ * @return the topology under the given domain, e.g. {["/group:1/zone:2/rack:0": {"/host:0",
+ * "/host:1"}, ["/group:1/zone:2/rack:1": {"/host:2", "/host:3"}]}
+ */
+ private Map<String, List<String>> getTopologyUnderDomain(Map<String, String> domainMap) {
+ LinkedHashMap<String, String> orderedDomain = validateAndOrderDomain(domainMap);
+ TrieNode startNode = _trieClusterTopology.getNode(orderedDomain);
+ Map<String, TrieNode> children = startNode.getChildren();
+ Map<String, List<String>> results = new HashMap<>();
+ children.entrySet().forEach(child -> {
+ results.put(startNode.getPath() + DELIMITER + child.getKey(),
+ truncatePath(_trieClusterTopology.getPathUnderNode(child.getValue()),
+ child.getValue().getPath()));
+ });
+ return results;
+ }
+
+ /**
+ * Return the full topology of a certain domain type.
+ * @param domainType a specific type of domain, e.g. zone
+ * @return the topology of the given domain type, e.g. {["/group:0/zone:0": {"rack:0/host:0",
+ * "rack:1/host:1"}, ["/group:0/zone:1": {"/rack:0:host:2", "/rack:1/host:3"}]}
+ */
+ private Map<String, List<String>> getTopologyUnderDomainType(String domainType) {
+ String[] topologyKeys = _trieClusterTopology.getTopologyKeys();
+ if (domainType.equals(topologyKeys[0])) {
+ return getTopologyMap();
+ }
+ Map<String, List<String>> results = new HashMap<>();
+ String parentDomainType = null;
+ for (int i = 1; i < topologyKeys.length; i++) {
+ if (topologyKeys[i].equals(domainType)) {
+ parentDomainType = topologyKeys[i - 1];
+ break;
+ }
+ }
+ // get all the starting nodes for the domain type
+ List<TrieNode> startNodes = _trieClusterTopology.getStartNodes(parentDomainType);
+ for (TrieNode startNode : startNodes) {
+ results.putAll(getTopologyUnderPath(startNode.getPath()));
+ }
+ return results;
+ }
+
+ /**
+ * Return the topology under a certain path as a map. The key of the returned map is the next
+ * level domain, and the value is a list of string that represents the path to each end node in
+ * that domain.
+ * @param path a path to a certain Trie node, e.g. /group:1/zone:2
+ * @return the topology under the given domain, e.g. {["/group:1/zone:2/rack:0": {"/host:0",
+ * "/host:1"}, ["/group:1/zone:2/rack:1": {"/host:2", "/host:3"}]}
+ */
+ private Map<String, List<String>> getTopologyUnderPath(String path) {
+ Map<String, String> domain = convertPathToDomain(path);
+ return getTopologyUnderDomain(domain);
+ }
+
+ /**
+ * Validate the domain provided has continuous fields in cluster topology definition. If it
+ * has, order the domain based on cluster topology definition. E.g. if the cluster topology is
+ * /group/zone/rack/instance, and domain is provided as {["zone": "1"], ["group", "2"]} will be
+ * reordered in a LinkedinHashMap as {["group", "2"], ["zone": "1"]}
+ */
+ private LinkedHashMap<String, String> validateAndOrderDomain(Map<String, String> domainMap) {
+ LinkedHashMap<String, String> orderedDomain = new LinkedHashMap<>();
+ if (domainMap == null) {
+ throw new IllegalArgumentException("The domain should not be null");
+ }
+ String[] topologyKeys = _trieClusterTopology.getTopologyKeys();
+ for (int i = 0; i < domainMap.size(); i++) {
+ if (!domainMap.containsKey(topologyKeys[i])) {
+ throw new IllegalArgumentException(String
+ .format("The input domain is not valid, the key %s is required", topologyKeys[i]));
+ } else {
+ orderedDomain.put(topologyKeys[i], domainMap.get(topologyKeys[i]));
+ }
+ }
+ return orderedDomain;
+ }
+
+ /**
+ * Truncate each path in the given set and only retain path starting from current node's
+ * children to each end node.
+ * @param toRemovePath The path from root to current node. It should be removed so that users
+ * can get a better view.
+ */
+ private List<String> truncatePath(Set<String> paths, String toRemovePath) {
+ List<String> results = new ArrayList<>();
+ paths.forEach(path -> {
+ String truncatedPath = path.replace(toRemovePath, "");
+ results.add(truncatedPath);
+ });
+ return results;
+ }
+
+ private Map<String, String> convertPathToDomain(String path) {
+ Map<String, String> results = new HashMap<>();
+ for (String part : path.substring(1).split(DELIMITER)) {
+ results.put(part.substring(0, part.indexOf(CONNECTOR)),
+ part.substring(part.indexOf(CONNECTOR) + 1));
+ }
+ return results;
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 8a0d72c..0fdc3b4 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -53,6 +53,7 @@ import org.apache.helix.PropertyKey.Builder;
import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
import org.apache.helix.ZNRecord;
+import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
@@ -1047,6 +1048,25 @@ public class ZKHelixAdmin implements HelixAdmin {
}
@Override
+ public ClusterTopology getClusterTopology(String clusterName) {
+ Map<String, InstanceConfig> instanceConfigMap = new HashMap<>();
+ String path = PropertyPathBuilder.instanceConfig(clusterName);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<>(_zkClient);
+ List<ZNRecord> znRecords = baseAccessor.getChildren(path, null, 0, 0, 0);
+ for (ZNRecord record : znRecords) {
+ if (record != null) {
+ InstanceConfig instanceConfig = new InstanceConfig(record);
+ instanceConfigMap.put(instanceConfig.getInstanceName(), instanceConfig);
+ }
+ }
+ path = PropertyPathBuilder.liveInstance(clusterName);
+ List<String> liveNodes = baseAccessor.getChildNames(path, 0);
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkClient);
+ ClusterConfig clusterConfig = configAccessor.getClusterConfig(clusterName);
+ return new ClusterTopology(liveNodes, instanceConfigMap, clusterConfig);
+ }
+
+ @Override
public List<String> getStateModelDefs(String clusterName) {
return _zkClient.getChildren(PropertyPathBuilder.stateModelDef(clusterName));
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/ClusterTrie.java b/helix-core/src/main/java/org/apache/helix/model/ClusterTrie.java
new file mode 100644
index 0000000..f1f91a4
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/ClusterTrie.java
@@ -0,0 +1,227 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.helix.HelixException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * This is a class that uses a trie data structure to represent cluster topology. Each node
+ * except the terminal node represents a certain domain in the topology, and an terminal node
+ * represents an instance in the cluster.
+ */
+public class ClusterTrie {
+ public static final String DELIMITER = "/";
+ public static final String CONNECTOR = ":";
+
+ private static Logger logger = LoggerFactory.getLogger(ClusterTrie.class);
+ private TrieNode _rootNode;
+ private String[] _topologyKeys;
+ private String _faultZoneType;
+ private List<String> _invalidInstances = new ArrayList<>();
+
+ public ClusterTrie(final List<String> liveNodes,
+ final Map<String, InstanceConfig> instanceConfigMap, ClusterConfig clusterConfig) {
+ validateInstanceConfig(liveNodes, instanceConfigMap);
+ _topologyKeys = getTopologyDef(clusterConfig);
+ _faultZoneType = clusterConfig.getFaultZoneType();
+ _invalidInstances = getInvalidInstancesFromConfig(instanceConfigMap, _topologyKeys);
+ instanceConfigMap.keySet().removeAll(_invalidInstances);
+ _rootNode = constructTrie(instanceConfigMap, _topologyKeys);
+ }
+
+ public TrieNode getRootNode() {
+ return _rootNode;
+ }
+
+ public String[] getTopologyKeys() {
+ return _topologyKeys;
+ }
+
+ public String getFaultZoneType() {
+ return _faultZoneType;
+ }
+
+ public List<String> getInvalidInstances() {
+ return _invalidInstances;
+ }
+
+ /**
+ * Return all the paths from a TrieNode as a set.
+ * @param node the node from where to collect all the nodes' paths.
+ * @return All the paths under the node.
+ */
+ public Set<String> getPathUnderNode(TrieNode node) {
+ Set<String> resultMap = new HashSet<>();
+ Deque<TrieNode> nodeStack = new ArrayDeque<>();
+ nodeStack.push(node);
+ while (!nodeStack.isEmpty()) {
+ node = nodeStack.pop();
+ if (node.getChildren().isEmpty()) {
+ resultMap.add(node.getPath());
+ } else {
+ for (TrieNode child : node.getChildren().values()) {
+ nodeStack.push(child);
+ }
+ }
+ }
+ return resultMap;
+ }
+
+ /**
+ * Get a specific node in the trie given a map of domain type and its value.
+ * @param domainMap a map of domain type and the corresponding value
+ * @return a trie node
+ */
+ public TrieNode getNode(LinkedHashMap<String, String> domainMap) {
+ TrieNode curNode = _rootNode;
+ TrieNode nextNode;
+ for (Map.Entry<String, String> entry : domainMap.entrySet()) {
+ nextNode = curNode.getChildren().get(entry.getKey() + CONNECTOR + entry.getValue());
+ if (nextNode == null) {
+ throw new IllegalArgumentException(String
+ .format("The input domain %s does not have the value %s", entry.getKey(),
+ entry.getValue()));
+ }
+ curNode = nextNode;
+ }
+ return curNode;
+ }
+
+ /**
+ * Get all the starting nodes for a certain domain type. E.g., if the domainType is "zone", it
+ * will return the list of trie nodes that represent zone:0, zone:1, zone:2, etc.
+ * @param domainType a specific domain type
+ * @return a list of trie nodes
+ */
+ public List<TrieNode> getStartNodes(String domainType) {
+ List<TrieNode> results = new ArrayList<>();
+ TrieNode curNode = _rootNode;
+ Deque<TrieNode> nodeStack = new ArrayDeque<>();
+ nodeStack.push(curNode);
+ while (!nodeStack.isEmpty()) {
+ curNode = nodeStack.pop();
+ if (curNode.getNodeKey().equals(domainType)) {
+ results.add(curNode);
+ } else {
+ for (TrieNode child : curNode.getChildren().values()) {
+ nodeStack.push(child);
+ }
+ }
+ }
+ return results;
+ }
+
+ private void validateInstanceConfig(final List<String> liveNodes,
+ final Map<String, InstanceConfig> instanceConfigMap) {
+ if (instanceConfigMap == null || !instanceConfigMap.keySet().containsAll(liveNodes)) {
+ List<String> liveNodesCopy = new ArrayList<>();
+ liveNodesCopy.addAll(liveNodes);
+ throw new HelixException(String.format("Config for instances %s is not found!",
+ instanceConfigMap == null ? liveNodes
+ : liveNodesCopy.removeAll(instanceConfigMap.keySet())));
+ }
+ }
+
+ private List<String> getInvalidInstancesFromConfig(Map<String, InstanceConfig> instanceConfigMap,
+ final String[] topologyKeys) {
+ List<String> invalidInstances = new ArrayList<>();
+ for (String instanceName : instanceConfigMap.keySet()) {
+ try {
+ Map<String, String> domainAsMap = instanceConfigMap.get(instanceName).getDomainAsMap();
+ for (String key : topologyKeys) {
+ String value = domainAsMap.get(key);
+ if (value == null || value.length() == 0) {
+ logger.info(String.format("Domain %s for instance %s is not set", domainAsMap.get(key),
+ instanceName));
+ invalidInstances.add(instanceName);
+ break;
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ invalidInstances.add(instanceName);
+ }
+ }
+ return invalidInstances;
+ }
+
+ // Note that we do not validate whether topology-aware is enabled or fault zone type is
+ // defined, as they do not block the construction of the trie
+ private String[] getTopologyDef(ClusterConfig clusterConfig) {
+ String[] topologyDef;
+ String topologyDefInConfig = clusterConfig.getTopology();
+ if (topologyDefInConfig == null || !topologyDefInConfig.trim().startsWith(DELIMITER)) {
+ throw new HelixException(String.format("The topology of cluster %s is invalid!",
+ clusterConfig.getClusterName()));
+ }
+ // A list of all keys in cluster topology, e.g., a cluster topology defined as
+ // /group/zone/rack/host will return ["group", "zone", "rack", "host"].
+ topologyDef =
+ Arrays.asList(topologyDefInConfig.split(DELIMITER)).stream().map(str -> str.trim())
+ .filter(str -> !str.isEmpty()).collect(Collectors.toList()).toArray(new String[0]);
+ if (topologyDef.length == 0) {
+ throw new HelixException(String.format("The topology of cluster %s is not correctly defined",
+ clusterConfig.getClusterName()));
+ }
+ return topologyDef;
+ }
+
+ /**
+ * Constructs a trie based on the provided instance config map. It loops through all instance
+ * configs and constructs the trie in a top down manner.
+ */
+ private TrieNode constructTrie(Map<String, InstanceConfig> instanceConfigMap,
+ final String[] topologyKeys) {
+ TrieNode rootNode = new TrieNode("", "ROOT");
+ Map<String, Map<String, String>> instanceDomainsMap = new HashMap<>();
+ instanceConfigMap.entrySet().forEach(
+ entry -> instanceDomainsMap.put(entry.getKey(), entry.getValue().getDomainAsMap()));
+
+ for (Map.Entry<String, Map<String, String>> entry : instanceDomainsMap.entrySet()) {
+ TrieNode curNode = rootNode;
+ StringBuilder path = new StringBuilder();
+ for (int i = 0; i < topologyKeys.length; i++) {
+ String key = topologyKeys[i] + CONNECTOR + entry.getValue().get(topologyKeys[i]);
+ path.append(DELIMITER).append(key);
+ TrieNode nextNode = curNode.getChildren().get(key);
+ if (nextNode == null) {
+ nextNode = new TrieNode(path.toString(), topologyKeys[i]);
+ }
+ curNode.addChild(key, nextNode);
+ curNode = nextNode;
+ }
+ }
+ return rootNode;
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/model/TrieNode.java b/helix-core/src/main/java/org/apache/helix/model/TrieNode.java
new file mode 100644
index 0000000..e58ae90
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/TrieNode.java
@@ -0,0 +1,56 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class TrieNode {
+ // A mapping between trie key and children nodes.
+ private Map<String, TrieNode> _children;
+
+ // the complete path/prefix leading to the current node.
+ private final String _path;
+
+ private final String _nodeKey;
+
+ TrieNode(String path, String nodeKey) {
+ _path = path;
+ _nodeKey = nodeKey;
+ _children = new HashMap<>();
+ }
+
+ public Map<String, TrieNode> getChildren() {
+ return _children;
+ }
+
+ public String getPath() {
+ return _path;
+ }
+
+ public String getNodeKey() {
+ return _nodeKey;
+ }
+
+ public void addChild(String key, TrieNode node) {
+ _children.put(key, node);
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index f7fe23c..0769558 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -23,8 +23,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
+
import org.apache.helix.BaseDataAccessor;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
@@ -39,9 +42,11 @@ import org.apache.helix.PropertyType;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.examples.MasterSlaveStateModelFactory;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -51,6 +56,7 @@ import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.model.builder.ConstraintItemBuilder;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
@@ -117,7 +123,7 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
try {
tool.addInstance(clusterName, config);
- Assert.fail("should fail if add an alredy-existing instance");
+ Assert.fail("should fail if add an already-existing instance");
} catch (HelixException e) {
// OK
}
@@ -574,4 +580,72 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk);
}
+
+ @Test
+ public void testGetDomainInformation() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setTopology("/group/zone/rack/host");
+ clusterConfig.setFaultZoneType("rack");
+
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ _configAccessor.setClusterConfig(clusterName, clusterConfig);
+
+ HelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<>(_gZkClient));
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+
+ for (int i = 0; i < 42; i++) {
+
+ String hostname = "myhost" + i;
+ String port = "9999";
+ String instanceName = hostname + "_" + port;
+ InstanceConfig instanceConfig = new InstanceConfig(instanceName);
+ instanceConfig.setHostName(hostname);
+ instanceConfig.setPort(port);
+ if (i == 40) {
+ instanceConfig.setDomain(String
+ .format("invaliddomain=%s,zone=%s,rack=%s,host=%s", "mygroup" + i % 2, "myzone" + i % 4,
+ "myrack" + i % 4, hostname));
+ } else if (i == 41) {
+ instanceConfig.setDomain("invaliddomain");
+ } else {
+ String domain = String
+ .format("group=%s,zone=%s,rack=%s,host=%s", "mygroup" + i % 2, "myzone" + i % 4,
+ "myrack" + i % 4, hostname);
+ instanceConfig.setDomain(domain);
+ }
+ LiveInstance liveInstance = new LiveInstance(instanceName);
+ liveInstance.setSessionId(UUID.randomUUID().toString());
+ liveInstance.setHelixVersion(UUID.randomUUID().toString());
+ accessor.setProperty(keyBuilder.liveInstance(instanceName), liveInstance);
+ admin.addInstance(clusterName, instanceConfig);
+ admin.enableInstance(clusterName, instanceName, true);
+ }
+
+ ClusterTopology clusterTopology = admin.getClusterTopology(clusterName);
+ Assert.assertNotNull(clusterTopology);
+ Map<String, List<String>> results = clusterTopology.getTopologyMap();
+ Assert.assertEquals(results.size(), 2);
+ Assert.assertTrue(results.containsKey("/group:mygroup0"));
+ Assert.assertTrue(results.containsKey("/group:mygroup1"));
+ Assert.assertEquals(results.get("/group:mygroup0").size(), 20);
+ Assert.assertEquals(results.get("/group:mygroup1").size(), 20);
+
+ results = clusterTopology.getFaultZoneMap();
+ Assert.assertEquals(results.size(), 4);
+ Assert.assertEquals(results.get("/group:mygroup0/zone:myzone0/rack:myrack0").size(), 10);
+ Assert.assertTrue(results.get("/group:mygroup0/zone:myzone0/rack:myrack0").contains("/host"
+ + ":myhost0"));
+
+ Assert.assertEquals(clusterTopology.getInvalidInstances().size(), 2);
+ Assert.assertTrue(clusterTopology.getInvalidInstances()
+ .containsAll(new HashSet<>(Arrays.asList("myhost40_9999", "myhost41_9999"))));
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index e06c902..19f0875 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -30,6 +30,7 @@ import org.apache.helix.HelixManager;
import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
import org.apache.helix.ZNRecord;
+import org.apache.helix.api.topology.ClusterTopology;
import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
@@ -323,6 +324,11 @@ public class MockHelixAdmin implements HelixAdmin {
}
+ @Override
+ public ClusterTopology getClusterTopology(String clusterName) {
+ return null;
+ }
+
@Override public List<String> getStateModelDefs(String clusterName) {
return null;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/TestClusterTrie.java b/helix-core/src/test/java/org/apache/helix/model/TestClusterTrie.java
new file mode 100644
index 0000000..29385af
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/TestClusterTrie.java
@@ -0,0 +1,141 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.HelixException;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestClusterTrie {
+ private ClusterTrie _trie;
+
+ final List<String> _instanceNames = new ArrayList<>();
+ final Map<String, InstanceConfig> _instanceConfigMap = new HashMap<>();
+ private ClusterConfig _clusterConfig;
+ final int _numOfNodes = 40;
+
+ @BeforeClass
+ public void beforeClass() {
+ for (int i = 0; i < _numOfNodes; i++) {
+ _instanceNames.add(String.valueOf(i));
+ }
+ createClusterConfig();
+ createInstanceConfigMap();
+ }
+
+ @Test
+ public void testConstructionMissingInstanceConfigMap() {
+ Map<String, InstanceConfig> emptyMap = new HashMap<>();
+ try {
+ new ClusterTrie(_instanceNames, emptyMap, _clusterConfig);
+ Assert.fail("Expecting instance config not found exception");
+ } catch (HelixException e) {
+ Assert.assertTrue(e.getMessage().contains("is not found!"));
+ }
+ }
+
+ @Test
+ public void testConstructionMissingTopology() {
+ _clusterConfig.setTopology(null);
+ try {
+ new ClusterTrie(_instanceNames, _instanceConfigMap, _clusterConfig);
+ Assert.fail("Expecting topology not set exception");
+ } catch (HelixException e) {
+ Assert.assertTrue(e.getMessage().contains("is invalid!"));
+ }
+ _clusterConfig.setTopology("/group/zone/rack/host");
+ }
+
+ @Test
+ public void testConstructionInvalidTopology() {
+ _clusterConfig.setTopology("invalidTopology");
+ try {
+ new ClusterTrie(_instanceNames, _instanceConfigMap, _clusterConfig);
+ Assert.fail("Expecting topology invalid exception");
+ } catch (HelixException e) {
+ Assert.assertTrue(e.getMessage().contains("is invalid!"));
+ }
+ _clusterConfig.setTopology("/group/zone/rack/host");
+ }
+
+ @Test
+ public void testConstructionNormal() {
+ try {
+ _trie = new ClusterTrie(_instanceNames, _instanceConfigMap, _clusterConfig);
+ } catch (HelixException e) {
+ Assert.fail("Not expecting HelixException");
+ }
+ }
+
+ @Test
+ public void testConstructionNormalWithSpace() {
+ _clusterConfig.setTopology("/ group/ zone/rack/host");
+ try {
+ _trie = new ClusterTrie(_instanceNames, _instanceConfigMap, _clusterConfig);
+ } catch (HelixException e) {
+ Assert.fail("Not expecting HelixException");
+ }
+ String[] topologyDef = _trie.getTopologyKeys();
+ Assert.assertEquals(topologyDef[0], "group");
+ Assert.assertEquals(topologyDef[1], "zone");
+ _clusterConfig.setTopology("/group/zone/rack/host");
+ }
+
+ @Test
+ public void testConstructionNormalWithInvalidConfig() {
+ String instance = "invalidInstance";
+ InstanceConfig config = new InstanceConfig(instance);
+ config.setDomain(String.format("invaliddomain=%s, zone=%s, rack=%s, host=%s", 1, 2, 3, 4));
+ _instanceConfigMap.put(instance, config);
+ try {
+ _trie = new ClusterTrie(_instanceNames, _instanceConfigMap, _clusterConfig);
+ } catch (HelixException e) {
+ Assert.fail("Not expecting HelixException");
+ }
+ Assert.assertEquals(_trie.getInvalidInstances().size(), 1);
+ Assert.assertEquals(_trie.getInvalidInstances().get(0), instance );
+ _instanceConfigMap.remove(instance);
+ }
+
+ private void createInstanceConfigMap() {
+ for (int i = 0; i < _instanceNames.size(); i++) {
+ String instance = _instanceNames.get(i);
+ InstanceConfig config = new InstanceConfig(instance);
+ // create 2 groups, 4 zones, and 4 racks.
+ config.setDomain(String.format("group=%s, zone=%s, rack=%s, host=%s", i % (_numOfNodes / 10),
+ i % (_numOfNodes / 5), i % (_numOfNodes / 5), instance));
+ _instanceConfigMap.put(instance, config);
+ }
+ }
+
+ private void createClusterConfig() {
+ _clusterConfig = new ClusterConfig("test");
+ _clusterConfig.setTopologyAwareEnabled(true);
+ _clusterConfig.setTopology("/group/zone/rack/host");
+ _clusterConfig.setFaultZoneType("rack");
+ }
+}
\ No newline at end of file
[helix] 15/18: Minor fix to add participant auto registration
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 35bbc7ca381548a9208bac48e1d73fd3545300b3
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Tue Aug 25 09:58:49 2020 -0700
Minor fix to add participant auto registration
---
.../main/java/org/apache/helix/ConfigAccessor.java | 10 ++++-----
.../org/apache/helix/HelixPropertyFactory.java | 10 ++++++++-
.../java/org/apache/helix/tools/ClusterSetup.java | 2 +-
.../java/org/apache/helix/TestConfigAccessor.java | 4 ++--
.../org/apache/helix/tools/TestClusterSetup.java | 6 +++---
.../helix/rest/server/TestClusterAccessor.java | 24 +++++++++-------------
6 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index da6edda..2efc61a 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -569,7 +569,7 @@ public class ConfigAccessor {
* @return The instance of {@link CloudConfig}
*/
public CloudConfig getCloudConfig(String clusterName) {
- if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
throw new HelixException(
String.format("Failed to get config. cluster: %s is not setup.", clusterName));
}
@@ -591,7 +591,7 @@ public class ConfigAccessor {
* @param cloudConfig
*/
public void deleteCloudConfigFields(String clusterName, CloudConfig cloudConfig) {
- if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
throw new HelixException("fail to delete cloud config. cluster: " + clusterName + " is NOT setup.");
}
@@ -610,7 +610,7 @@ public class ConfigAccessor {
}
private void updateCloudConfig(String clusterName, CloudConfig cloudConfig, boolean overwrite) {
- if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
throw new HelixException("Fail to update cloud config. cluster: " + clusterName + " is NOT setup.");
}
@@ -619,9 +619,9 @@ public class ConfigAccessor {
String zkPath = scope.getZkPath();
if (overwrite) {
- ZKUtil.createOrReplace(_zkClient, zkPath, cloudConfig.getRecord(), true);
+ ZKUtil.createOrReplace(zkClient, zkPath, cloudConfig.getRecord(), true);
} else {
- ZKUtil.createOrUpdate(_zkClient, zkPath, cloudConfig.getRecord(), true, true);
+ ZKUtil.createOrUpdate(zkClient, zkPath, cloudConfig.getRecord(), true, true);
}
}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
index fa394a2..e111f3b 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
@@ -22,6 +22,10 @@ package org.apache.helix;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
+
+import org.apache.helix.manager.zk.ZNRecordSerializer;
+import org.apache.helix.manager.zk.client.DedicatedZkClientFactory;
+import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.model.CloudConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +51,11 @@ public final class HelixPropertyFactory {
* Clients may override these values.
*/
public HelixManagerProperty getHelixManagerProperty(String zkAddress, String clusterName) {
- ConfigAccessor configAccessor = new ConfigAccessor(zkAddress);
+ HelixZkClient.ZkClientConfig clientConfig = new HelixZkClient.ZkClientConfig();
+ clientConfig.setZkSerializer(new ZNRecordSerializer());
+ HelixZkClient zkClient = DedicatedZkClientFactory.getInstance()
+ .buildZkClient(new HelixZkClient.ZkConnectionConfig(zkAddress), clientConfig);
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
CloudConfig cloudConfig;
// The try-catch logic is for backward compatibility reason only. Even if the cluster is not set
// up yet, constructing a new ZKHelixManager should not throw an exception
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 9663223..803f8a5 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -175,7 +175,7 @@ public class ClusterSetup {
// If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
if (cloudConfig.isCloudEnabled()
&& cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
- ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkClient);
ClusterConfig clusterConfig = new ClusterConfig(clusterName);
clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
clusterConfig.setTopologyAwareEnabled(true);
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index 02b5a45..74fae40 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -216,7 +216,7 @@ public class TestConfigAccessor extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -260,7 +260,7 @@ public class TestConfigAccessor extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 1772521..446bb0a 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -356,7 +356,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -422,7 +422,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -506,7 +506,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index bcf0d0c..3dfb883 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -27,7 +27,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@@ -39,19 +38,19 @@ import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.cloud.azure.AzureConstants;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.integration.manager.ClusterDistributedController;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZKUtil;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
-import org.apache.helix.model.CloudConfig;
-import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.rest.common.HelixRestNamespace;
import org.apache.helix.rest.server.auditlog.AuditLog;
import org.apache.helix.rest.server.resources.AbstractResource;
@@ -66,9 +65,6 @@ import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import com.google.common.collect.ImmutableMap;
-import com.sun.research.ws.wadl.HTTPMethods;
-
public class TestClusterAccessor extends AbstractTestClass {
@BeforeClass
@@ -591,7 +587,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -671,7 +667,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -701,7 +697,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -760,7 +756,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -809,14 +805,14 @@ public class TestClusterAccessor extends AbstractTestClass {
Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been created
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNotNull(cloudConfigFromZk);
String urlBase = "clusters/" + clusterName + "/cloudconfig/";
delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- _configAccessor = new ConfigAccessor(ZK_ADDR);
+ _configAccessor = new ConfigAccessor(_gZkClient);
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk);
@@ -847,7 +843,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been created
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNotNull(cloudConfigFromZk);
@@ -860,7 +856,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- _configAccessor = new ConfigAccessor(ZK_ADDR);
+ _configAccessor = new ConfigAccessor(_gZkClient);
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk.getCloudID());
Assert.assertNull(cloudConfigFromZk.getCloudProvider());
[helix] 06/18: Implement Azure cloud instance information processor
(#698)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit da32e4ee26e764063af24b2b86be182123778b9f
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Fri Feb 7 15:24:29 2020 -0800
Implement Azure cloud instance information processor (#698)
Implement Azure cloud instance information processor. The processor performs the functions of fetching and parsing instance information from Azure cloud environment. The endpoint that the processor queries is a globally standard url, called Azure Instance Metadata Service.
---
helix-core/pom.xml | 5 +
.../cloud/CloudInstanceInformationProcessor.java | 1 -
.../cloud/azure/AzureCloudInstanceInformation.java | 11 +-
.../AzureCloudInstanceInformationProcessor.java | 113 +++++++++++++++++++--
.../org/apache/helix/cloud/MockHttpClient.java | 53 ++++++++++
...TestAzureCloudInstanceInformationProcessor.java | 69 +++++++++++++
helix-core/src/test/resources/AzureResponse.json | 104 +++++++++++++++++++
helix-rest/pom.xml | 5 -
8 files changed, 343 insertions(+), 18 deletions(-)
diff --git a/helix-core/pom.xml b/helix-core/pom.xml
index 8288d8f..95aefe3 100644
--- a/helix-core/pom.xml
+++ b/helix-core/pom.xml
@@ -159,6 +159,11 @@ under the License.
<artifactId>metrics-core</artifactId>
<version>3.2.3</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.8</version>
+ </dependency>
</dependencies>
<build>
<resources>
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
index a365777..9ff6b00 100644
--- a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
@@ -21,7 +21,6 @@ package org.apache.helix.api.cloud;
import java.util.List;
-
/**
* Generic interface to fetch and parse cloud instance information
*/
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
index f7fd657..1fef205 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -19,17 +19,17 @@ package org.apache.helix.cloud.azure;
* under the License.
*/
+import java.util.HashMap;
import java.util.Map;
import org.apache.helix.api.cloud.CloudInstanceInformation;
-
public class AzureCloudInstanceInformation implements CloudInstanceInformation {
private Map<String, String> _cloudInstanceInfoMap;
/**
* Instantiate the AzureCloudInstanceInformation using each field individually.
- * Users should use AzureCloudInstanceInformation.Builder to create information.
+ * Users should use AzureCloudInstanceInformation.Builder to set field information.
* @param cloudInstanceInfoMap
*/
protected AzureCloudInstanceInformation(Map<String, String> cloudInstanceInfoMap) {
@@ -42,10 +42,11 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
}
public static class Builder {
- private Map<String, String> _cloudInstanceInfoMap = null;
+
+ private final Map<String, String> _cloudInstanceInfoMap = new HashMap<>();
public AzureCloudInstanceInformation build() {
- return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
+ return new AzureCloudInstanceInformation(new HashMap<>(_cloudInstanceInfoMap));
}
public Builder setInstanceName(String name) {
@@ -68,4 +69,4 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
return this;
}
}
-}
\ No newline at end of file
+}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 84a102c..85e2bb5 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -19,38 +19,137 @@ package org.apache.helix.cloud.azure;
* under the License.
*/
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
+import javax.net.ssl.SSLException;
+import org.apache.helix.HelixCloudProperty;
+import org.apache.helix.HelixException;
import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.EntityUtils;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+public class AzureCloudInstanceInformationProcessor
+ implements CloudInstanceInformationProcessor<String> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AzureCloudInstanceInformationProcessor.class);
+ private final CloseableHttpClient _closeableHttpClient;
+ private final HelixCloudProperty _helixCloudProperty;
+ private final String COMPUTE = "compute";
+ private final String INSTANCE_NAME = "vmId";
+ private final String DOMAIN = "platformFaultDomain";
+ private final String INSTANCE_SET_NAME = "vmScaleSetName";
-public class AzureCloudInstanceInformationProcessor implements CloudInstanceInformationProcessor<String> {
+ public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty) {
+ _helixCloudProperty = helixCloudProperty;
- public AzureCloudInstanceInformationProcessor() {
+ RequestConfig requestConifg = RequestConfig.custom()
+ .setConnectionRequestTimeout((int) helixCloudProperty.getCloudRequestTimeout())
+ .setConnectTimeout((int) helixCloudProperty.getCloudConnectionTimeout()).build();
+
+ HttpRequestRetryHandler httpRequestRetryHandler =
+ (IOException exception, int executionCount, HttpContext context) -> {
+ LOG.warn("Execution count: " + executionCount + ".", exception);
+ return !(executionCount >= helixCloudProperty.getCloudMaxRetry()
+ || exception instanceof InterruptedIOException
+ || exception instanceof UnknownHostException || exception instanceof SSLException);
+ };
+
+ //TODO: we should regularize the way how httpClient should be used throughout Helix. e.g. Helix-rest could also use in the same way
+ _closeableHttpClient = HttpClients.custom().setDefaultRequestConfig(requestConifg)
+ .setRetryHandler(httpRequestRetryHandler).build();
}
/**
- * fetch the raw Azure cloud instance information
+ * This constructor is for unit test purpose only.
+ * User could provide helixCloudProperty and a mocked http client to test the functionality of
+ * this class.
+ */
+ public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty,
+ CloseableHttpClient closeableHttpClient) {
+ _helixCloudProperty = helixCloudProperty;
+ _closeableHttpClient = closeableHttpClient;
+ }
+
+ /**
+ * Fetch raw Azure cloud instance information based on the urls provided
* @return raw Azure cloud instance information
*/
@Override
public List<String> fetchCloudInstanceInformation() {
List<String> response = new ArrayList<>();
- //TODO: implement the fetching logic
+ for (String url : _helixCloudProperty.getCloudInfoSources()) {
+ response.add(getAzureCloudInformationFromUrl(url));
+ }
return response;
}
/**
+ * Query Azure Instance Metadata Service to get the instance(VM) information
+ * @return raw Azure cloud instance information
+ */
+ private String getAzureCloudInformationFromUrl(String url) {
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setHeader("Metadata", "true");
+
+ try {
+ CloseableHttpResponse response = _closeableHttpClient.execute(httpGet);
+ if (response == null || response.getStatusLine().getStatusCode() != 200) {
+ String errorMsg = String.format(
+ "Failed to get an HTTP Response for the request. Response: {}. Status code: {}",
+ (response == null ? "NULL" : response.getStatusLine().getReasonPhrase()),
+ response.getStatusLine().getStatusCode());
+ throw new HelixException(errorMsg);
+ }
+ String responseString = EntityUtils.toString(response.getEntity());
+ LOG.info("VM instance information query result: {}", responseString);
+ return responseString;
+ } catch (IOException e) {
+ throw new HelixException(
+ String.format("Failed to get Azure cloud instance information from url {}", url), e);
+ }
+ }
+
+ /**
* Parse raw Azure cloud instance information.
* @return required azure cloud instance information
*/
@Override
public AzureCloudInstanceInformation parseCloudInstanceInformation(List<String> responses) {
AzureCloudInstanceInformation azureCloudInstanceInformation = null;
- //TODO: implement the parsing logic
+ if (responses.size() > 1) {
+ throw new HelixException("Multiple responses are not supported for Azure now");
+ }
+ String response = responses.get(0);
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ JsonNode jsonNode = mapper.readTree(response);
+ JsonNode computeNode = jsonNode.path(COMPUTE);
+ if (!computeNode.isMissingNode()) {
+ String vmName = computeNode.path(INSTANCE_NAME).getTextValue();
+ String platformFaultDomain = computeNode.path(DOMAIN).getTextValue();
+ String vmssName = computeNode.path(INSTANCE_SET_NAME).getValueAsText();
+ AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
+ builder.setInstanceName(vmName).setFaultDomain(platformFaultDomain)
+ .setInstanceSetName(vmssName);
+ azureCloudInstanceInformation = builder.build();
+ }
+ } catch (IOException e) {
+ throw new HelixException(String.format("Error in parsing cloud instance information: {}", response, e));
+ }
return azureCloudInstanceInformation;
}
}
-
-
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java b/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
new file mode 100644
index 0000000..03e687e
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
@@ -0,0 +1,53 @@
+package org.apache.helix.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.InputStream;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+
+/**
+ * Mock a http client and provide response using resource file. This is for unit test purpose only.
+ */
+public class MockHttpClient {
+ protected CloseableHttpClient createMockHttpClient(String file) throws Exception {
+ InputStream responseInputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(file);
+ HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+ StatusLine statusLine = Mockito.mock(StatusLine.class);
+
+ CloseableHttpResponse mockCloseableHttpResponse = Mockito.mock(CloseableHttpResponse.class);
+ CloseableHttpClient mockCloseableHttpClient = Mockito.mock(CloseableHttpClient.class);
+
+ Mockito.when(httpEntity.getContent()).thenReturn(responseInputStream);
+ Mockito.when(mockCloseableHttpClient.execute(Matchers.any(HttpGet.class))).thenReturn(mockCloseableHttpResponse);
+ Mockito.when(mockCloseableHttpResponse.getEntity()).thenReturn(httpEntity);
+ Mockito.when(mockCloseableHttpResponse.getStatusLine()).thenReturn(statusLine);
+ Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+
+ return mockCloseableHttpClient;
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..10ba42d
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -0,0 +1,69 @@
+package org.apache.helix.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+import org.apache.helix.HelixCloudProperty;
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+import org.apache.helix.cloud.azure.AzureCloudInstanceInformation;
+import org.apache.helix.cloud.azure.AzureCloudInstanceInformationProcessor;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.CloudConfig;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+/**
+ * Unit test for {@link AzureCloudInstanceInformationProcessor}
+ */
+public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
+
+ @Test()
+ public void testAzureCloudInstanceInformationProcessing() throws Exception {
+ String responseFile = "AzureResponse.json";
+
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
+ HelixCloudProperty helixCloudProperty = new HelixCloudProperty(cloudConfigBuilder.build());
+ AzureCloudInstanceInformationProcessor processor = new AzureCloudInstanceInformationProcessor(
+ helixCloudProperty, createMockHttpClient(responseFile));
+ List<String> response = processor.fetchCloudInstanceInformation();
+
+ Assert.assertEquals(response.size(), 1);
+ Assert.assertNotNull(response.get(0));
+
+ // Verify the response from mock http client
+ AzureCloudInstanceInformation azureCloudInstanceInformation =
+ processor.parseCloudInstanceInformation(response);
+ Assert.assertEquals(azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()), "2");
+ Assert.assertEquals(
+ azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()),
+ "test-helix");
+ Assert.assertEquals(
+ azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
+ "d2b921cc-c16c-41f7-a86d-a445eac6ec26");
+ }
+}
diff --git a/helix-core/src/test/resources/AzureResponse.json b/helix-core/src/test/resources/AzureResponse.json
new file mode 100644
index 0000000..dfe13ad
--- /dev/null
+++ b/helix-core/src/test/resources/AzureResponse.json
@@ -0,0 +1,104 @@
+{
+ "compute": {
+ "azEnvironment": "AzurePublicCloud",
+ "customData": "",
+ "location": "southcentralus",
+ "name": "test-helix_1",
+ "offer": "",
+ "osType": "Linux",
+ "placementGroupId": "81e605b2-a807-48ee-a84a-63c76a9c9543",
+ "plan": {
+ "name": "",
+ "product": "",
+ "publisher": ""
+ },
+ "platformFaultDomain": "2",
+ "platformUpdateDomain": "2",
+ "provider": "Microsoft.Compute",
+ "publicKeys": [],
+ "publisher": "",
+ "resourceGroupName": "scus-lpsazureei1-app-rg",
+ "resourceId": "/subscriptions/c9a251d8-1272-4c0f-8055-8271bbc1d677/resourceGroups/scus-lpsazureei1-app-rg/providers/Microsoft.Compute/virtualMachines/test-helix_2",
+ "sku": "",
+ "storageProfile": {
+ "dataDisks": [],
+ "imageReference": {
+ "id": "/subscriptions/7dd5a659-67c4-441c-ac0b-d48b7a029668/resourceGroups/scus-infra-app-rg/providers/Microsoft.Compute/galleries/pieimagerepo/images/FastCOP4/versions/190924.1.1",
+ "offer": "",
+ "publisher": "",
+ "sku": "",
+ "version": ""
+ },
+ "osDisk": {
+ "caching": "ReadWrite",
+ "createOption": "FromImage",
+ "diskSizeGB": "32",
+ "encryptionSettings": {
+ "enabled": "false"
+ },
+ "image": {
+ "uri": ""
+ },
+ "managedDisk": {
+ "id": "/subscriptions/c9a251d8-1272-4c0f-8055-8271bbc1d677/resourceGroups/scus-lpsazureei1-app-rg/providers/Microsoft.Compute/disks/test-helix_test-helix_2_OsDisk_1_124c3534b8e848e296ec22b24d44c027",
+ "storageAccountType": "Standard_LRS"
+ },
+ "name": "test-helix_test-helix_2_OsDisk_1_124c3534b8e848e296ec22b24d44c027",
+ "osType": "Linux",
+ "vhd": {
+ "uri": ""
+ },
+ "writeAcceleratorEnabled": "false"
+ }
+ },
+ "subscriptionId": "c9a251d8-1272-4c0f-8055-8271bbc1d677",
+ "tags": "automation:terraform;environment:dev;module:lid-vmss;moduleVersion:0.0.1",
+ "tagsList": [
+ {
+ "name": "automation",
+ "value": "terraform"
+ },
+ {
+ "name": "environment",
+ "value": "dev"
+ },
+ {
+ "name": "module",
+ "value": "lid-vmss"
+ },
+ {
+ "name": "moduleVersion",
+ "value": "0.0.1"
+ }
+ ],
+ "version": "",
+ "vmId": "d2b921cc-c16c-41f7-a86d-a445eac6ec26",
+ "vmScaleSetName": "test-helix",
+ "vmSize": "Standard_D16s_v3",
+ "zone": ""
+ },
+ "network": {
+ "interface": [
+ {
+ "ipv4": {
+ "ipAddress": [
+ {
+ "privateIpAddress": "10.3.64.12",
+ "publicIpAddress": ""
+ }
+ ],
+ "subnet": [
+ {
+ "address": "10.3.64.0",
+ "prefix": "19"
+ }
+ ]
+ },
+ "ipv6": {
+ "ipAddress": []
+ },
+ "macAddress": "000D3A769010"
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml
index 44550b7..76db93c 100644
--- a/helix-rest/pom.xml
+++ b/helix-rest/pom.xml
@@ -67,11 +67,6 @@ under the License.
<version>3.8.1</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.8</version>
- </dependency>
- <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.1.0.RC0</version>
[helix] 07/18: Modify participant manager to add cluster auto
registration logic (#695)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ffe55cc42183256b0a3ac8751b94508c9184ffa8
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Fri Feb 7 15:41:36 2020 -0800
Modify participant manager to add cluster auto registration logic (#695)
Modify participant logic to add logic for instance auto registration to a Helix cluster.
Auto registration differs with existing auto join in that auto registration will retrieve the fault domain information for the instance and populate it under instanceConfig. With auto registration on, user does not need to manually populate instance information in Zookeeper.
---
.../helix/manager/zk/ParticipantManager.java | 100 ++++++++++++++++-----
.../main/java/org/apache/helix/util/HelixUtil.java | 19 ++++
.../manager/MockParticipantManager.java | 10 ++-
3 files changed, 107 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index f25f29e..7ae62b8 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -20,6 +20,8 @@ package org.apache.helix.manager.zk;
*/
import java.lang.management.ManagementFactory;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -31,6 +33,7 @@ import org.apache.helix.AccessOption;
import org.apache.helix.BaseDataAccessor;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixCloudProperty;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerProperty;
@@ -40,6 +43,8 @@ import org.apache.helix.PreConnectCallback;
import org.apache.helix.PropertyKey;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZNRecordBucketizer;
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.messaging.DefaultMessagingService;
import org.apache.helix.model.CurrentState;
@@ -52,6 +57,7 @@ import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.helix.participant.StateMachineEngine;
import org.apache.helix.participant.statemachine.ScheduledTaskStateModelFactory;
+import org.apache.helix.util.HelixUtil;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +67,7 @@ import org.slf4j.LoggerFactory;
*/
public class ParticipantManager {
private static Logger LOG = LoggerFactory.getLogger(ParticipantManager.class);
+ private static final String CLOUD_PROCESSOR_PATH_PREFIX = "org.apache.helix.cloud.";
final HelixZkClient _zkclient;
final HelixManager _manager;
@@ -107,7 +114,7 @@ public class ParticipantManager {
}
/**
- * Handle new session for a participang.
+ * Handle new session for a participant.
* @throws Exception
*/
public void handleNewSession() throws Exception {
@@ -132,18 +139,31 @@ public class ParticipantManager {
}
private void joinCluster() {
- // Read cluster config and see if instance can auto join the cluster
+ // Read cluster config and see if an instance can auto join or auto register to the cluster
boolean autoJoin = false;
+ boolean autoRegistration = false;
+
+ // Read "allowParticipantAutoJoin" flag to see if an instance can auto join to the cluster
try {
- HelixConfigScope scope =
- new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(
- _manager.getClusterName()).build();
- autoJoin =
- Boolean.parseBoolean(_configAccessor.get(scope,
- ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN));
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
+ .forCluster(_manager.getClusterName()).build();
+ autoJoin = Boolean
+ .parseBoolean(_configAccessor.get(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN));
LOG.info("instance: " + _instanceName + " auto-joining " + _clusterName + " is " + autoJoin);
} catch (Exception e) {
- // autoJoin is false
+ LOG.info("auto join is false for cluster" + _clusterName);
+ }
+
+ // Read cloud config and see if an instance can auto register to the cluster
+ // Difference between auto join and auto registration is that the latter will also populate the
+ // domain information in instance config
+ try {
+ autoRegistration =
+ Boolean.valueOf(_helixManagerProperty.getHelixCloudProperty().getCloudEnabled());
+ LOG.info("instance: " + _instanceName + " auto-register " + _clusterName + " is "
+ + autoRegistration);
+ } catch (Exception e) {
+ LOG.info("auto registration is false for cluster" + _clusterName);
}
if (!ZKUtil.isInstanceSetup(_zkclient, _clusterName, _instanceName, _instanceType)) {
@@ -151,23 +171,61 @@ public class ParticipantManager {
throw new HelixException("Initial cluster structure is not set up for instance: "
+ _instanceName + ", instanceType: " + _instanceType);
} else {
- LOG.info(_instanceName + " is auto-joining cluster: " + _clusterName);
- InstanceConfig instanceConfig = new InstanceConfig(_instanceName);
- String hostName = _instanceName;
- String port = "";
- int lastPos = _instanceName.lastIndexOf("_");
- if (lastPos > 0) {
- hostName = _instanceName.substring(0, lastPos);
- port = _instanceName.substring(lastPos + 1);
+ if (!autoRegistration) {
+ LOG.info(_instanceName + " is auto-joining cluster: " + _clusterName);
+ _helixAdmin.addInstance(_clusterName, HelixUtil.composeInstanceConfig(_instanceName));
+ } else {
+ LOG.info(_instanceName + " is auto-registering cluster: " + _clusterName);
+ CloudInstanceInformation cloudInstanceInformation = getCloudInstanceInformation();
+ String domain = cloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name());
+
+ // Disable the verification for now
+ /*String cloudIdInRemote = cloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name());
+ String cloudIdInConfig = _configAccessor.getCloudConfig(_clusterName).getCloudID();
+
+ // validate that the instance is auto registering to the correct cluster
+ if (!cloudIdInRemote.equals(cloudIdInConfig)) {
+ throw new IllegalArgumentException(String.format(
+ "cloudId in config: %s is not consistent with cloudId from remote: %s. The instance is auto registering to a wrong cluster.",
+ cloudIdInConfig, cloudIdInRemote));
+ }*/
+
+ InstanceConfig instanceConfig = HelixUtil.composeInstanceConfig(_instanceName);
+ instanceConfig.setDomain(domain);
+ _helixAdmin.addInstance(_clusterName, instanceConfig);
}
- instanceConfig.setHostName(hostName);
- instanceConfig.setPort(port);
- instanceConfig.setInstanceEnabled(true);
- _helixAdmin.addInstance(_clusterName, instanceConfig);
}
}
}
+ private CloudInstanceInformation getCloudInstanceInformation() {
+ String cloudInstanceInformationProcessorName =
+ _helixManagerProperty.getHelixCloudProperty().getCloudInfoProcessorName();
+ try {
+ // fetch cloud instance information for the instance
+ String cloudInstanceInformationProcessorClassName = CLOUD_PROCESSOR_PATH_PREFIX
+ + _helixManagerProperty.getHelixCloudProperty().getCloudProvider().toLowerCase() + "."
+ + cloudInstanceInformationProcessorName;
+ Class processorClass = Class.forName(cloudInstanceInformationProcessorClassName);
+ Constructor constructor = processorClass.getConstructor(HelixCloudProperty.class);
+ CloudInstanceInformationProcessor processor = (CloudInstanceInformationProcessor) constructor
+ .newInstance(_helixManagerProperty.getHelixCloudProperty());
+ List<String> responses = processor.fetchCloudInstanceInformation();
+
+ // parse cloud instance information for the participant
+ CloudInstanceInformation cloudInstanceInformation =
+ processor.parseCloudInstanceInformation(responses);
+ return cloudInstanceInformation;
+ } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException
+ | IllegalAccessException | InvocationTargetException ex) {
+ throw new HelixException(
+ "Failed to create a new instance for the class: " + cloudInstanceInformationProcessorName,
+ ex);
+ }
+ }
+
private void createLiveInstance() {
String liveInstancePath = _keyBuilder.liveInstance(_instanceName).getPath();
LiveInstance liveInstance = new LiveInstance(_instanceName);
diff --git a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
index 5e7f7b4..de8fcf6 100644
--- a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
@@ -290,4 +290,23 @@ public final class HelixUtil {
return propertyDefaultValue;
}
+ /**
+ * Compose the config for an instance
+ * @param instanceName
+ * @return InstanceConfig
+ */
+ public static InstanceConfig composeInstanceConfig(String instanceName) {
+ InstanceConfig instanceConfig = new InstanceConfig(instanceName);
+ String hostName = instanceName;
+ String port = "";
+ int lastPos = instanceName.lastIndexOf("_");
+ if (lastPos > 0) {
+ hostName = instanceName.substring(0, lastPos);
+ port = instanceName.substring(lastPos + 1);
+ }
+ instanceConfig.setHostName(hostName);
+ instanceConfig.setPort(port);
+ instanceConfig.setInstanceEnabled(true);
+ return instanceConfig;
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
index d1677c6..c2a446e 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
@@ -21,6 +21,7 @@ package org.apache.helix.integration.manager;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import org.apache.helix.HelixCloudProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.manager.zk.CallbackHandler;
import org.apache.helix.manager.zk.ZKHelixManager;
@@ -47,6 +48,7 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
protected MockMSModelFactory _msModelFactory;
protected DummyLeaderStandbyStateModelFactory _lsModelFactory;
protected DummyOnlineOfflineStateModelFactory _ofModelFactory;
+ protected HelixCloudProperty _helixCloudProperty;
public MockParticipantManager(String zkAddr, String clusterName, String instanceName) {
this(zkAddr, clusterName, instanceName, 10);
@@ -54,11 +56,17 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
public MockParticipantManager(String zkAddr, String clusterName, String instanceName,
int transDelay) {
+ this(zkAddr, clusterName, instanceName, transDelay, null);
+ }
+
+ public MockParticipantManager(String zkAddr, String clusterName, String instanceName,
+ int transDelay, HelixCloudProperty helixCloudProperty) {
super(clusterName, instanceName, InstanceType.PARTICIPANT, zkAddr);
_transDelay = transDelay;
_msModelFactory = new MockMSModelFactory(null);
_lsModelFactory = new DummyLeaderStandbyStateModelFactory(_transDelay);
_ofModelFactory = new DummyOnlineOfflineStateModelFactory(_transDelay);
+ _helixCloudProperty = helixCloudProperty;
}
public void setTransition(MockTransition transition) {
@@ -135,4 +143,4 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
public List<CallbackHandler> getHandlers() {
return _handlers;
}
-}
+}
\ No newline at end of file
[helix] 12/18: Add REST and JAVA API to modify existing cloudconfig
(#898)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 980fbd1487d27b9f2cf223d4938b1c429e5fbc4f
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Mar 17 13:07:46 2020 -0700
Add REST and JAVA API to modify existing cloudconfig (#898)
The new APIs are added to update/delete cloudconfig entries.
Tests have been modified accordingly.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 42 +++++++++++++++-
.../AzureCloudInstanceInformationProcessor.java | 8 ++--
.../java/org/apache/helix/model/CloudConfig.java | 2 +-
.../java/org/apache/helix/TestConfigAccessor.java | 56 ++++++++++++++++++++--
...TestAzureCloudInstanceInformationProcessor.java | 8 ++--
.../server/resources/helix/ClusterAccessor.java | 13 +++--
.../helix/rest/server/TestClusterAccessor.java | 50 ++++++++++++++++++-
7 files changed, 157 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index f1e3ffa..da6edda 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -582,7 +582,47 @@ public class ConfigAccessor {
return null;
}
- return new CloudConfig.Builder(record).build();
+ return new CloudConfig(record);
+ }
+
+ /**
+ * Delete cloud config fields (not the whole config)
+ * @param clusterName
+ * @param cloudConfig
+ */
+ public void deleteCloudConfigFields(String clusterName, CloudConfig cloudConfig) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("fail to delete cloud config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ remove(scope, cloudConfig.getRecord());
+ }
+
+ /**
+ * Update cloud config
+ * @param clusterName
+ * @param cloudConfig
+ */
+ public void updateCloudConfig(String clusterName, CloudConfig cloudConfig) {
+ updateCloudConfig(clusterName, cloudConfig, false);
+ }
+
+ private void updateCloudConfig(String clusterName, CloudConfig cloudConfig, boolean overwrite) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("Fail to update cloud config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ String zkPath = scope.getZkPath();
+
+ if (overwrite) {
+ ZKUtil.createOrReplace(_zkClient, zkPath, cloudConfig.getRecord(), true);
+ } else {
+ ZKUtil.createOrUpdate(_zkClient, zkPath, cloudConfig.getRecord(), true, true);
+ }
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 464ff55..f0f75f7 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -145,15 +145,15 @@ public class AzureCloudInstanceInformationProcessor
String azureTopology = AzureConstants.AZURE_TOPOLOGY;
String[] parts = azureTopology.trim().split("/");
//The hostname will be filled in by each participant
- String domain = parts[0] + "=" + platformFaultDomain + "," + parts[1] + "=";
+ String domain = parts[1] + "=" + platformFaultDomain + "," + parts[2] + "=";
AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
- builder.setInstanceName(vmName).setFaultDomain(domain)
- .setInstanceSetName(vmssName);
+ builder.setInstanceName(vmName).setFaultDomain(domain).setInstanceSetName(vmssName);
azureCloudInstanceInformation = builder.build();
}
} catch (IOException e) {
- throw new HelixException(String.format("Error in parsing cloud instance information: {}", response, e));
+ throw new HelixException(
+ String.format("Error in parsing cloud instance information: {}", response, e));
}
return azureCloudInstanceInformation;
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index 79c7330..89fcd43 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -64,7 +64,7 @@ public class CloudConfig extends HelixProperty {
* The constructor from the ZNRecord.
* @param record
*/
- private CloudConfig(ZNRecord record) {
+ public CloudConfig(ZNRecord record) {
super(CLOUD_CONFIG_KW);
_record.setSimpleFields(record.getSimpleFields());
_record.setListFields(record.getListFields());
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index d59beb6..02b5a45 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -216,7 +216,7 @@ public class TestConfigAccessor extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -226,10 +226,10 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
// Change the processor name and check if processor name has been changed in Zookeeper.
- cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2");
- cloudConfigInit = cloudConfigInitBuilder.build();
- ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
- admin.addCloudConfig(clusterName, cloudConfigInit);
+ CloudConfig.Builder cloudConfigToUpdateBuilder = new CloudConfig.Builder();
+ cloudConfigToUpdateBuilder.setCloudInfoProcessorName("TestProcessor2");
+ CloudConfig cloudConfigToUpdate = cloudConfigToUpdateBuilder.build();
+ _configAccessor.updateCloudConfig(clusterName, cloudConfigToUpdate);
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
@@ -239,4 +239,50 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
}
+
+ @Test
+ public void testDeleteCloudConfig() throws Exception {
+ ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR);
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestCloudID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE);
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Change the processor name and check if processor name has been changed in Zookeeper.
+ CloudConfig.Builder cloudConfigBuilderToDelete = new CloudConfig.Builder();
+ cloudConfigBuilderToDelete.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigBuilderToDelete.setCloudID("TestCloudID");
+ CloudConfig cloudConfigToDelete = cloudConfigBuilderToDelete.build();
+
+ _configAccessor.deleteCloudConfigFields(clusterName, cloudConfigToDelete);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertNull(cloudConfigFromZk.getCloudID());
+ listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
index 10ba42d..350c9a9 100644
--- a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -55,12 +55,12 @@ public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
// Verify the response from mock http client
AzureCloudInstanceInformation azureCloudInstanceInformation =
processor.parseCloudInstanceInformation(response);
- Assert.assertEquals(azureCloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()), "2");
Assert.assertEquals(
azureCloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()),
- "test-helix");
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()),
+ "faultDomain=2," + "hostname=");
+ Assert.assertEquals(azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()), "test-helix");
Assert.assertEquals(
azureCloudInstanceInformation
.get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 4f5bb20..f8e1d86 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -604,6 +604,7 @@ public class ClusterAccessor extends AbstractHelixResource {
return notFound();
}
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
// Here to update cloud config
Command command;
if (commandStr == null || commandStr.isEmpty()) {
@@ -616,22 +617,24 @@ public class ClusterAccessor extends AbstractHelixResource {
}
}
- HelixAdmin admin = getHelixAdmin();
-
ZNRecord record;
+ CloudConfig cloudConfig;
try {
record = toZNRecord(content);
+ cloudConfig = new CloudConfig(record);
} catch (IOException e) {
_logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
return badRequest("Input is not a vaild ZNRecord!");
}
try {
switch (command) {
+ case delete: {
+ configAccessor.deleteCloudConfigFields(clusterId, cloudConfig);
+ }
+ break;
case update: {
try {
- CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
- admin.removeCloudConfig(clusterId);
- admin.addCloudConfig(clusterId, cloudConfig);
+ configAccessor.updateCloudConfig(clusterId, cloudConfig);
} catch (HelixException ex) {
_logger.error("Error in updating a CloudConfig to cluster: " + clusterId, ex);
return badRequest(ex.getMessage());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 5bb29ce..bcf0d0c 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -796,7 +796,7 @@ public class TestClusterAccessor extends AbstractTestClass {
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
-
+
ZNRecord record = new ZNRecord("testZnode");
record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
@@ -813,7 +813,6 @@ public class TestClusterAccessor extends AbstractTestClass {
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNotNull(cloudConfigFromZk);
String urlBase = "clusters/" + clusterName + "/cloudconfig/";
-
delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
@@ -824,7 +823,54 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+
@Test(dependsOnMethods = "testDeleteCloudConfig")
+ public void testPartialDeleteCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+
+ ZNRecord record = new ZNRecord(clusterName);
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), "TestProcessor");
+ _gSetupTool.addCluster(clusterName, true, new CloudConfig.Builder(record).build());
+
+ String urlBase = "clusters/" + clusterName +"/cloudconfig/";
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+ // Read CloudConfig from Zookeeper and make sure it has been created
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNotNull(cloudConfigFromZk);
+
+ record = new ZNRecord(clusterName);
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", Command.delete.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(), CloudProvider.AZURE.name());
+ post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and make sure it has been removed
+ _configAccessor = new ConfigAccessor(ZK_ADDR);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNull(cloudConfigFromZk.getCloudID());
+ Assert.assertNull(cloudConfigFromZk.getCloudProvider());
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(),"TestProcessor");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testPartialDeleteCloudConfig")
public void testUpdateCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
_gSetupTool.addCluster("TestCloud", true);
[helix] 17/18: Add REST API for cluster topology (#1416)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 790e158788295295455e934ec588ffd87f5eac94
Author: Meng Zhang <mn...@linkedin.com>
AuthorDate: Tue Oct 6 22:09:52 2020 -0700
Add REST API for cluster topology (#1416)
This commit provides a few REST endpoints for user to retrieve cluster topology information. The APIs are added in ClusterAccessor.
---
.../server/resources/helix/ClusterAccessor.java | 26 ++++++
.../helix/rest/server/AbstractTestClass.java | 5 +-
.../helix/rest/server/TestClusterAccessor.java | 97 ++++++++++++++++++++++
3 files changed, 126 insertions(+), 2 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index ccbe12a..d0a4997 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -300,6 +300,32 @@ public class ClusterAccessor extends AbstractHelixResource {
return OK(objectMapper.writeValueAsString(clusterTopology));
}
+ @GET
+ @Path("{clusterId}/topologymap")
+ public Response getClusterTopologyMap(@PathParam("clusterId") String clusterId) {
+ HelixAdmin admin = getHelixAdmin();
+ Map<String, List<String>> topologyMap;
+ try {
+ topologyMap = admin.getClusterTopology(clusterId).getTopologyMap();
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+ return JSONRepresentation(topologyMap);
+ }
+
+ @GET
+ @Path("{clusterId}/faultzonemap")
+ public Response getClusterFaultZoneMap(@PathParam("clusterId") String clusterId) {
+ HelixAdmin admin = getHelixAdmin();
+ Map<String, List<String>> faultZoneMap;
+ try {
+ faultZoneMap = admin.getClusterTopology(clusterId).getFaultZoneMap();
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+ return JSONRepresentation(faultZoneMap);
+ }
+
@POST
@Path("{clusterId}/configs")
public Response updateClusterConfig(
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
index 347be89..5e73c37 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/AbstractTestClass.java
@@ -473,8 +473,9 @@ public class AbstractTestClass extends JerseyTestNg.ContainerPerClassTest {
final Response response = webTarget.request().get();
Assert.assertEquals(response.getStatus(), expectedReturnStatus);
- // NOT_FOUND will throw text based html
- if (expectedReturnStatus != Response.Status.NOT_FOUND.getStatusCode()) {
+ // NOT_FOUND and BAD_REQUEST will throw text based html
+ if (expectedReturnStatus != Response.Status.NOT_FOUND.getStatusCode()
+ && expectedReturnStatus != Response.Status.BAD_REQUEST.getStatusCode()) {
Assert.assertEquals(response.getMediaType().getType(), "application");
} else {
Assert.assertEquals(response.getMediaType().getType(), "text");
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 3dfb883..06a02c1 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -121,6 +122,102 @@ public class TestClusterAccessor extends AbstractTestClass {
}
@Test(dependsOnMethods = "testGetClusterTopology")
+ public void testGetClusterTopologyAndFaultZoneMap() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String topologyMapUrlBase = "clusters/TestCluster_1/topologymap/";
+ String faultZoneUrlBase = "clusters/TestCluster_1/faultzonemap/";
+
+ // test invalid case where instance config and cluster topology have not been set.
+ get(topologyMapUrlBase, null, Response.Status.BAD_REQUEST.getStatusCode(), true);
+ get(faultZoneUrlBase, null, Response.Status.BAD_REQUEST.getStatusCode(), true);
+
+ String cluster = "TestCluster_1";
+ for (int i = 0; i < 5; i++) {
+ String instance = cluster + "localhost_129" + String.valueOf(18 + i);
+ HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(cluster, _baseAccessor);
+ InstanceConfig instanceConfig =
+ helixDataAccessor.getProperty(helixDataAccessor.keyBuilder().instanceConfig(instance));
+ instanceConfig.setDomain("helixZoneId=zone0,instance=" + instance);
+ helixDataAccessor
+ .setProperty(helixDataAccessor.keyBuilder().instanceConfig(instance), instanceConfig);
+ }
+
+ for (int i = 0; i < 5; i++) {
+ String instance = cluster + "localhost_129" + String.valueOf(23 + i);
+ HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(cluster, _baseAccessor);
+ InstanceConfig instanceConfig =
+ helixDataAccessor.getProperty(helixDataAccessor.keyBuilder().instanceConfig(instance));
+ instanceConfig.setDomain("helixZoneId=zone1,instance=" + instance);
+ helixDataAccessor
+ .setProperty(helixDataAccessor.keyBuilder().instanceConfig(instance), instanceConfig);
+ }
+
+ // test invalid case where instance config is set, but cluster topology has not been set.
+ get(topologyMapUrlBase, null, Response.Status.BAD_REQUEST.getStatusCode(), true);
+ get(faultZoneUrlBase, null, Response.Status.BAD_REQUEST.getStatusCode(), true);
+
+ ClusterConfig configDelta = new ClusterConfig(cluster);
+ configDelta.getRecord().setSimpleField("TOPOLOGY", "/helixZoneId/instance");
+ updateClusterConfigFromRest(cluster, configDelta, Command.update);
+
+ //get valid cluster topology map
+ String topologyMapDef = get(topologyMapUrlBase, null, Response.Status.OK.getStatusCode(), true);
+ Map<String, Object> topologyMap =
+ OBJECT_MAPPER.readValue(topologyMapDef, new TypeReference<HashMap<String, Object>>() {
+ });
+ Assert.assertEquals(topologyMap.size(), 2);
+ Assert.assertTrue(topologyMap.get("/helixZoneId:zone0") instanceof List);
+ List<String> instances = (List<String>) topologyMap.get("/helixZoneId:zone0");
+ Assert.assertEquals(instances.size(), 5);
+ Assert.assertTrue(instances.containsAll(new HashSet<>(Arrays
+ .asList("/instance:TestCluster_1localhost_12918",
+ "/instance:TestCluster_1localhost_12919",
+ "/instance:TestCluster_1localhost_12920",
+ "/instance:TestCluster_1localhost_12921",
+ "/instance:TestCluster_1localhost_12922"))));
+
+ Assert.assertTrue(topologyMap.get("/helixZoneId:zone1") instanceof List);
+ instances = (List<String>) topologyMap.get("/helixZoneId:zone1");
+ Assert.assertEquals(instances.size(), 5);
+ Assert.assertTrue(instances.containsAll(new HashSet<>(Arrays
+ .asList("/instance:TestCluster_1localhost_12923",
+ "/instance:TestCluster_1localhost_12924",
+ "/instance:TestCluster_1localhost_12925",
+ "/instance:TestCluster_1localhost_12926",
+ "/instance:TestCluster_1localhost_12927"))));
+
+ configDelta = new ClusterConfig(cluster);
+ configDelta.getRecord().setSimpleField("FAULT_ZONE_TYPE", "helixZoneId");
+ updateClusterConfigFromRest(cluster, configDelta, Command.update);
+
+ //get valid cluster fault zone map
+ String faultZoneMapDef = get(faultZoneUrlBase, null, Response.Status.OK.getStatusCode(), true);
+ Map<String, Object> faultZoneMap =
+ OBJECT_MAPPER.readValue(faultZoneMapDef, new TypeReference<HashMap<String, Object>>() {
+ });
+ Assert.assertEquals(faultZoneMap.size(), 2);
+ Assert.assertTrue(faultZoneMap.get("/helixZoneId:zone0") instanceof List);
+ instances = (List<String>) faultZoneMap.get("/helixZoneId:zone0");
+ Assert.assertEquals(instances.size(), 5);
+ Assert.assertTrue(instances.containsAll(new HashSet<>(Arrays
+ .asList("/instance:TestCluster_1localhost_12918",
+ "/instance:TestCluster_1localhost_12919",
+ "/instance:TestCluster_1localhost_12920",
+ "/instance:TestCluster_1localhost_12921",
+ "/instance:TestCluster_1localhost_12922"))));
+
+ Assert.assertTrue(faultZoneMap.get("/helixZoneId:zone1") instanceof List);
+ instances = (List<String>) faultZoneMap.get("/helixZoneId:zone1");
+ Assert.assertEquals(instances.size(), 5);
+ Assert.assertTrue(instances.containsAll(new HashSet<>(Arrays
+ .asList("/instance:TestCluster_1localhost_12923",
+ "/instance:TestCluster_1localhost_12924",
+ "/instance:TestCluster_1localhost_12925",
+ "/instance:TestCluster_1localhost_12926",
+ "/instance:TestCluster_1localhost_12927"))));
+ }
+
+ @Test(dependsOnMethods = "testGetClusterTopologyAndFaultZoneMap")
public void testAddConfigFields() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
String cluster = _clusters.iterator().next();
[helix] 02/18: add Helix cloud interface and implementation
skeleton methods
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 4286e2f15b04cb14cba4153392fbfa23a8b462df
Author: Meng Zhang <mn...@mnzhang-ld2.linkedin.biz>
AuthorDate: Wed Nov 13 09:43:04 2019 -0800
add Helix cloud interface and implementation skeleton methods
---
.../helix/api/cloud/CloudInstanceInformation.java | 40 +++++++++++
.../cloud/CloudInstanceInformationProcessor.java | 41 ++++++++++++
.../cloud/azure/AzureCloudInstanceInformation.java | 77 ++++++++++++++++++++++
.../AzureCloudInstanceInformationProcessor.java | 56 ++++++++++++++++
4 files changed, 214 insertions(+)
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformation.java
new file mode 100644
index 0000000..b2429f9
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformation.java
@@ -0,0 +1,40 @@
+package org.apache.helix.api.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Generic interface for cloud instance information
+ */
+public interface CloudInstanceInformation {
+ /**
+ * Get the the value of a specific cloud instance field by name
+ * @return the value of the field
+ */
+ String get(String key);
+
+ /**
+ * The enum contains all the required cloud instance field in Helix
+ */
+ enum CloudInstanceField {
+ INSTANCE_NAME,
+ FAULT_DOMAIN,
+ INSTANCE_SET_NAME
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..a365777
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
@@ -0,0 +1,41 @@
+package org.apache.helix.api.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+
+/**
+ * Generic interface to fetch and parse cloud instance information
+ */
+public interface CloudInstanceInformationProcessor<T extends Object> {
+
+ /**
+ * Get the raw cloud instance information
+ * @return raw cloud instance information
+ */
+ List<T> fetchCloudInstanceInformation();
+
+ /**
+ * Parse the raw cloud instance information in responses and compose required cloud instance information
+ * @return required cloud instance information
+ */
+ CloudInstanceInformation parseCloudInstanceInformation(List<T> responses);
+}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
new file mode 100644
index 0000000..511f3e3
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -0,0 +1,77 @@
+package org.apache.helix.cloud.azure;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Map;
+
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+
+
+public class AzureCloudInstanceInformation implements CloudInstanceInformation {
+ private Map<String, String> _cloudInstanceInfoMap;
+
+ /**
+ * Instantiate the AzureCloudInstanceInformation using each field individually.
+ * Users should use AzureCloudInstanceInformation.Builder to create information.
+ * @param cloudInstanceInfoMap
+ */
+ protected AzureCloudInstanceInformation(Map<String, String> cloudInstanceInfoMap) {
+ _cloudInstanceInfoMap = cloudInstanceInfoMap;
+ }
+
+ @Override
+ public String get(String key) {
+ return _cloudInstanceInfoMap.get(key);
+ }
+
+ public static class Builder {
+ private Map<String, String> _cloudInstanceInfoMap = null;
+
+ public AzureCloudInstanceInformation build() {
+ return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
+ }
+
+ /**
+ * Default constructor
+ */
+ public Builder() {
+ }
+
+ public Builder setInstanceName(String v) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), v);
+ return this;
+ }
+
+ public Builder setFaultDomain(String v) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), v);
+ return this;
+ }
+
+ public Builder setInstanceSetName(String v) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), v);
+ return this;
+ }
+
+ public Builder setCloudInstanceInfoField(String key, String value) {
+ _cloudInstanceInfoMap.put(key, value);
+ return this;
+ }
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..84a102c
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -0,0 +1,56 @@
+package org.apache.helix.cloud.azure;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
+
+
+public class AzureCloudInstanceInformationProcessor implements CloudInstanceInformationProcessor<String> {
+
+ public AzureCloudInstanceInformationProcessor() {
+ }
+
+ /**
+ * fetch the raw Azure cloud instance information
+ * @return raw Azure cloud instance information
+ */
+ @Override
+ public List<String> fetchCloudInstanceInformation() {
+ List<String> response = new ArrayList<>();
+ //TODO: implement the fetching logic
+ return response;
+ }
+
+ /**
+ * Parse raw Azure cloud instance information.
+ * @return required azure cloud instance information
+ */
+ @Override
+ public AzureCloudInstanceInformation parseCloudInstanceInformation(List<String> responses) {
+ AzureCloudInstanceInformation azureCloudInstanceInformation = null;
+ //TODO: implement the parsing logic
+ return azureCloudInstanceInformation;
+ }
+}
+
+
[helix] 08/18: add one more test for auto registration (#806)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit c80fdb3f8ede319ed54982b3ad42a6379d2947ea
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Wed Feb 26 10:05:41 2020 -0800
add one more test for auto registration (#806)
Add a test case for participant auto registers to cluster in cloud environment. Note that the final step is expected to fail as the test is not running in a cloud environment.
---
.../paticipant/TestInstanceAutoJoin.java | 62 +++++++++++++++++++---
1 file changed, 54 insertions(+), 8 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
index f0ffc63..5d5ec46 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
@@ -1,16 +1,22 @@
package org.apache.helix.integration.paticipant;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.manager.zk.ZKHelixManager;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.IdealState.RebalanceMode;
import org.apache.helix.model.builder.ConfigScopeBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
+import static org.testng.Assert.*;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -32,14 +38,15 @@ import org.testng.annotations.Test;
public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
String db2 = TEST_DB + "2";
+ String db3 = TEST_DB + "3";
@Test
public void testInstanceAutoJoin() throws Exception {
HelixManager manager = _participants[0];
HelixDataAccessor accessor = manager.getHelixDataAccessor();
- _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "OnlineOffline", RebalanceMode.FULL_AUTO
- + "");
+ _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "OnlineOffline",
+ RebalanceMode.FULL_AUTO + "");
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 1);
String instance2 = "localhost_279699";
@@ -50,8 +57,8 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
Thread.sleep(500);
// Assert.assertFalse(result._thread.isAlive());
- Assert.assertTrue(null == manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2)));
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2)));
ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
@@ -63,15 +70,54 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
Thread.sleep(500);
// Assert.assertTrue(result._thread.isAlive() || result2._thread.isAlive());
for (int i = 0; i < 20; i++) {
- if (null == manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2))) {
+ if (null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2))) {
Thread.sleep(100);
} else
break;
}
- Assert.assertTrue(null != manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2)));
+ Assert.assertTrue(null != manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2)));
newParticipant.syncStop();
}
+
+ @Test(dependsOnMethods = "testInstanceAutoJoin")
+ public void testAutoRegistrationShouldFailWhenWaitingResponse() throws Exception {
+ // Create CloudConfig object and add to config
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
+ CloudConfig cloudConfig = cloudConfigBuilder.build();
+
+ HelixManager manager = _participants[0];
+ HelixDataAccessor accessor = manager.getHelixDataAccessor();
+
+ _gSetupTool.addResourceToCluster(CLUSTER_NAME, db3, 60, "OnlineOffline",
+ RebalanceMode.FULL_AUTO + "");
+ _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db3, 1);
+ String instance3 = "localhost_279700";
+
+ ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
+
+ manager.getConfigAccessor().set(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, "true");
+ // Write the CloudConfig to Zookeeper
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ MockParticipantManager autoParticipant =
+ new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance3);
+ autoParticipant.syncStart();
+
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance3)));
+ try {
+ manager.getConfigAccessor().getInstanceConfig(CLUSTER_NAME, instance3);
+ fail(
+ "Exception should be thrown because the instance cannot be added to the cluster due to the disconnection with Azure endpoint");
+ } catch (HelixException e) {
+
+ }
+ }
}
[helix] 13/18: Fix ClusterAccessor::createCluster wrt CloudConfig
(#937)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit acebdae24618d07f5d8f027c0107fe588168f11d
Author: Hunter Lee <hu...@linkedin.com>
AuthorDate: Wed Apr 8 15:33:21 2020 -0700
Fix ClusterAccessor::createCluster wrt CloudConfig (#937)
The logic for ClusterAccessor was faulty in that it called createCluster() twice. This PR fixes this.
---
.../server/resources/helix/ClusterAccessor.java | 29 +++++++++++-----------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index f8e1d86..ccbe12a 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -137,10 +137,10 @@ public class ClusterAccessor extends AbstractHelixResource {
public Response createCluster(@PathParam("clusterId") String clusterId,
@DefaultValue("false") @QueryParam("recreate") String recreate,
@DefaultValue("false") @QueryParam("addCloudConfig") String addCloudConfig,
- String content) {
+ String cloudConfigManifest) {
- boolean recreateIfExists = Boolean.valueOf(recreate);
- boolean cloudConfigIncluded = Boolean.valueOf(addCloudConfig);
+ boolean recreateIfExists = Boolean.parseBoolean(recreate);
+ boolean cloudConfigIncluded = Boolean.parseBoolean(addCloudConfig);
ClusterSetup clusterSetup = getClusterSetup();
@@ -150,20 +150,21 @@ public class ClusterAccessor extends AbstractHelixResource {
if (cloudConfigIncluded) {
ZNRecord record;
try {
- record = toZNRecord(content);
- } catch (IOException e) {
- _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
- return badRequest("Input is not a vaild ZNRecord!");
- }
- try {
+ record = toZNRecord(cloudConfigManifest);
cloudConfig = new CloudConfig.Builder(record).build();
- clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
- } catch (Exception ex) {
- _logger.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
- return badRequest(ex.getMessage());
+ } catch (IOException | HelixException e) {
+ String errMsg = "Failed to generate a valid CloudConfig from " + cloudConfigManifest;
+ _logger.error(errMsg, e);
+ return badRequest(errMsg + " Exception: " + e.getMessage());
}
}
- clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
+
+ try {
+ clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
+ } catch (Exception ex) {
+ _logger.error("Failed to create cluster {}. Exception: {}.", clusterId, ex);
+ return serverError(ex);
+ }
return created();
}
[helix] 03/18: Add java API to create cluster with CloudConfig
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit eb6c51a21431e6071de6e24543fdafd845bccbb7
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Nov 26 15:26:10 2019 -0800
Add java API to create cluster with CloudConfig
In this commit the below APIs have been added.
1- AddCluster with CloudCOnfig API.
2- addCloudConfig to existing cluster.
3- Update CloudConfig to update existing Cloud Config.
Several tests have been added to test these APIs.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 4 +-
.../src/main/java/org/apache/helix/HelixAdmin.java | 14 ++
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++++
.../java/org/apache/helix/model/CloudConfig.java | 149 +++++++--------------
.../java/org/apache/helix/tools/ClusterSetup.java | 17 ++-
.../java/org/apache/helix/TestConfigAccessor.java | 66 +++++++--
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 69 ++++++++++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 11 ++
.../apache/helix/model/cloud/TestCloudConfig.java | 60 ++++++---
.../org/apache/helix/tools/TestClusterSetup.java | 88 +++++++++++-
10 files changed, 368 insertions(+), 138 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 9bd84d1..d61b4ee 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -569,7 +569,7 @@ public class ConfigAccessor {
* @return The instance of {@link CloudConfig}
*/
public CloudConfig getCloudConfig(String clusterName) {
- if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException(
String.format("Failed to get config. cluster: %s is not setup.", clusterName));
}
@@ -582,7 +582,7 @@ public class ConfigAccessor {
return null;
}
- return new CloudConfig(record);
+ return new CloudConfig.Builder(record).build();
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index 7402c19..bb5f3bf 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -22,6 +22,7 @@ package org.apache.helix;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
@@ -380,6 +381,19 @@ public interface HelixAdmin {
void dropResource(String clusterName, String resourceName);
/**
+ * Add cloud config to the cluster.
+ * @param clusterName
+ * @param cloudConfig
+ */
+ void addCloudConfig(String clusterName, CloudConfig cloudConfig);
+
+ /**
+ * Remove the Cloud Config for specific cluster
+ * @param clusterName
+ */
+ void removeCloudConfig(String clusterName);
+
+ /**
* Get a list of state model definitions in a cluster
* @param clusterName
* @return
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index af0035a..8a0d72c 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -58,6 +58,7 @@ import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.manager.zk.client.SharedZkClientFactory;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -1019,6 +1020,33 @@ public class ZKHelixAdmin implements HelixAdmin {
}
@Override
+ public void addCloudConfig(String clusterName, CloudConfig cloudConfig) {
+ logger.info("Add CloudConfig to cluster {}, CloudConfig is {}.", clusterName,
+ cloudConfig.toString());
+
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("cluster " + clusterName + " is not setup yet");
+ }
+
+ CloudConfig.Builder builder = new CloudConfig.Builder(cloudConfig);
+ CloudConfig cloudConfigBuilder = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfigBuilder);
+ }
+
+ @Override
+ public void removeCloudConfig(String clusterName) {
+ logger.info("Remove Cloud Config for cluster {}.", clusterName);
+ HelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.removeProperty(keyBuilder.cloudConfig());
+ }
+
+ @Override
public List<String> getStateModelDefs(String clusterName) {
return _zkClient.getChildren(PropertyPathBuilder.stateModelDef(clusterName));
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index c8ab6eb..f6c279c 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -30,6 +30,9 @@ import org.apache.helix.cloud.constants.CloudProvider;
* Cloud configurations
*/
public class CloudConfig extends HelixProperty {
+
+ public static final String CLOUD_CONFIG_KW = "CloudConfig";
+
/**
* Configurable characteristics of a cloud.
* NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
@@ -52,39 +55,22 @@ public class CloudConfig extends HelixProperty {
/**
* Instantiate the CloudConfig for the cloud
- * @param cluster
*/
- public CloudConfig(String cluster) {
- super(cluster);
+ private CloudConfig() {
+ super(CLOUD_CONFIG_KW);
}
/**
- * Instantiate with a pre-populated record
- * @param record a ZNRecord corresponding to a cloud configuration
+ * The constructor from the ZNRecord.
+ * @param record
*/
- public CloudConfig(ZNRecord record) {
- super(record);
+ private CloudConfig(ZNRecord record) {
+ super(CLOUD_CONFIG_KW);
+ _record.setSimpleFields(record.getSimpleFields());
+ _record.setListFields(record.getListFields());
+ _record.setMapFields(record.getMapFields());
}
- /**
- * Instantiate the config using each field individually.
- * Users should use CloudConfig.Builder to create CloudConfig.
- * @param cluster
- * @param enabled
- * @param cloudID
- */
- public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
- List<String> cloudInfoSource, String cloudProcessorName) {
- super(cluster);
- _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
- _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
- _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
- if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
- _record
- .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
- _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
- }
- }
/**
* Enable/Disable the CLOUD_ENABLED field.
@@ -135,15 +121,6 @@ public class CloudConfig extends HelixProperty {
}
/**
- * Set the CLOUD_INFO_PROCESSOR_NAME field.
- * @param cloudInfoProcessorName
- */
- public void setCloudInfoFProcessorName(String cloudInfoProcessorName) {
- _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
- cloudInfoProcessorName);
- }
-
- /**
* Get the CLOUD_INFO_PROCESSOR_NAME field.
* @return CLOUD_INFO_PROCESSOR_NAME field.
*/
@@ -152,14 +129,6 @@ public class CloudConfig extends HelixProperty {
}
/**
- * Set the CLOUD_PROVIDER field.
- * @param cloudProvider
- */
- public void setCloudProvider(CloudProvider cloudProvider) {
- _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
- }
-
- /**
* Get the CLOUD_PROVIDER field.
* @return CLOUD_PROVIDER field.
*/
@@ -167,32 +136,28 @@ public class CloudConfig extends HelixProperty {
return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
}
+
public static class Builder {
- private String _clusterName = null;
- private CloudProvider _cloudProvider;
- private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED;
- private String _cloudID;
- private List<String> _cloudInfoSources;
- private String _cloudInfoProcessorName;
+ private ZNRecord _record;
public CloudConfig build() {
validate();
- return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID,
- _cloudInfoSources, _cloudInfoProcessorName);
+ return new CloudConfig(_record);
}
/**
* Default constructor
*/
public Builder() {
+ _record = new ZNRecord(CLOUD_CONFIG_KW);
}
/**
- * Constructor with Cluster Name as input
- * @param clusterName
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a cloud configuration
*/
- public Builder(String clusterName) {
- _clusterName = clusterName;
+ public Builder(ZNRecord record) {
+ _record = record;
}
/**
@@ -200,89 +165,75 @@ public class CloudConfig extends HelixProperty {
* @param cloudConfig
*/
public Builder(CloudConfig cloudConfig) {
- _cloudEnabled = cloudConfig.isCloudEnabled();
- _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider());
- _cloudID = cloudConfig.getCloudID();
- _cloudInfoSources = cloudConfig.getCloudInfoSources();
- _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName();
- }
-
- public Builder setClusterName(String v) {
- _clusterName = v;
- return this;
+ _record = cloudConfig.getRecord();
}
public Builder setCloudEnabled(boolean isEnabled) {
- _cloudEnabled = isEnabled;
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), isEnabled);
return this;
}
public Builder setCloudProvider(CloudProvider cloudProvider) {
- _cloudProvider = cloudProvider;
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
return this;
}
- public Builder setCloudID(String v) {
- _cloudID = v;
+ public Builder setCloudID(String cloudID) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
return this;
}
- public Builder setCloudInfoSources(List<String> v) {
- _cloudInfoSources = v;
+ public Builder setCloudInfoSources(List<String> cloudInfoSources) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources);
return this;
}
- public Builder addCloudInfoSource(String v) {
- if (_cloudInfoSources == null) {
- _cloudInfoSources = new ArrayList<String>();
+ public Builder addCloudInfoSource(String cloudInfoSource) {
+ if (_record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()) == null) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), new ArrayList<String>());
}
- _cloudInfoSources.add(v);
+ List<String> cloudInfoSourcesList = _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
+ cloudInfoSourcesList.add(cloudInfoSource);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSourcesList);
return this;
}
- public Builder setCloudInfoProcessorName(String v) {
- _cloudInfoProcessorName = v;
+ public Builder setCloudInfoProcessorName(String cloudInfoProcessorName) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ cloudInfoProcessorName);
return this;
}
- public String getClusterName() {
- return _clusterName;
- }
-
- public CloudProvider getCloudProvider() {
- return _cloudProvider;
+ public String getCloudProvider() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
}
public boolean getCloudEnabled() {
- return _cloudEnabled;
+ return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(),
+ DEFAULT_CLOUD_ENABLED);
}
public String getCloudID() {
- return _cloudID;
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name());
}
public List<String> getCloudInfoSources() {
- return _cloudInfoSources;
+ return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
}
public String getCloudInfoProcessorName() {
- return _cloudInfoProcessorName;
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name());
}
private void validate() {
- if (_cloudEnabled) {
- if (_cloudID == null) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. The CloudID is missing from the config.");
- }
- if (_cloudProvider == null) {
+ if (this.getCloudProvider() == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
+ if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
+ || this.getCloudInfoSources().size() == 0) {
throw new HelixException(
- "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
- } else if (_cloudProvider == CloudProvider.CUSTOMIZED) {
- if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
- }
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
}
}
}
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 677c1b4..27d8c88 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -40,7 +40,9 @@ import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
+import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
@@ -48,6 +50,7 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.manager.zk.client.HelixZkClient;
import org.apache.helix.manager.zk.client.SharedZkClientFactory;
import org.apache.helix.model.BuiltInStateModelDefinitions;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -158,15 +161,23 @@ public class ClusterSetup {
_admin = zkHelixAdmin;
}
- public void addCluster(String clusterName, boolean overwritePrevious) {
+ public void addCluster(String clusterName, boolean overwritePrevious, CloudConfig cloudConfig)
+ throws HelixException {
_admin.addCluster(clusterName, overwritePrevious);
-
for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) {
addStateModelDef(clusterName, def.getStateModelDefinition().getId(),
- def.getStateModelDefinition(), overwritePrevious);
+ def.getStateModelDefinition(), overwritePrevious);
+ }
+
+ if (cloudConfig != null) {
+ _admin.addCloudConfig(clusterName, cloudConfig);
}
}
+ public void addCluster(String clusterName, boolean overwritePrevious) {
+ addCluster(clusterName, overwritePrevious, null);
+ }
+
public void activateCluster(String clusterName, String grandCluster, boolean enable) {
if (enable) {
_admin.addClusterToGrandCluster(clusterName, grandCluster);
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index f4377d6..d59beb6 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -19,13 +19,17 @@ package org.apache.helix;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.builder.ConfigScopeBuilder;
+import org.apache.helix.tools.ClusterSetup;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -60,9 +64,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(resourceConfigValue, "resourceConfigValue");
// partition scope config
- ConfigScope partitionScope =
- new ConfigScopeBuilder().forCluster(clusterName).forResource("testResource")
- .forPartition("testPartition").build();
+ ConfigScope partitionScope = new ConfigScopeBuilder().forCluster(clusterName)
+ .forResource("testResource").forPartition("testPartition").build();
configAccessor.set(partitionScope, "partitionConfigKey", "partitionConfigValue");
String partitionConfigValue = configAccessor.get(partitionScope, "partitionConfigKey");
Assert.assertEquals(partitionConfigValue, "partitionConfigValue");
@@ -105,9 +108,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
"should be [HELIX_ENABLED, HELIX_ENABLED_TIMESTAMP, HELIX_HOST, HELIX_PORT, participantConfigKey]");
Assert.assertEquals(keys.get(4), "participantConfigKey");
- keys =
- configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource",
- "testPartition");
+ keys = configAccessor.getKeys(ConfigScopeProperty.PARTITION, clusterName, "testResource",
+ "testPartition");
Assert.assertEquals(keys.size(), 1, "should be [partitionConfigKey]");
Assert.assertEquals(keys.get(0), "partitionConfigKey");
@@ -173,8 +175,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
try {
configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue");
- Assert
- .fail("Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet");
+ Assert.fail(
+ "Except fail to set participant-config because participant: localhost_12918 is not added to cluster yet");
} catch (HelixException e) {
// OK
}
@@ -183,8 +185,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
try {
configAccessor.set(participantScope, "participantConfigKey", "participantConfigValue");
} catch (Exception e) {
- Assert
- .fail("Except succeed to set participant-config because participant: localhost_12918 has been added to cluster");
+ Assert.fail(
+ "Except succeed to set participant-config because participant: localhost_12918 has been added to cluster");
}
String participantConfigValue = configAccessor.get(participantScope, "participantConfigKey");
@@ -193,4 +195,48 @@ public class TestConfigAccessor extends ZkUnitTestBase {
admin.dropCluster(clusterName);
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
+
+ @Test
+ public void testUpdateCloudConfig() throws Exception {
+ ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR);
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestCloudID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+
+ // Change the processor name and check if processor name has been changed in Zookeeper.
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2");
+ cloudConfigInit = cloudConfigInitBuilder.build();
+ ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCloudConfig(clusterName, cloudConfigInit);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index d372d67..f7fe23c 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.helix.BaseDataAccessor;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
@@ -38,7 +39,9 @@ import org.apache.helix.PropertyType;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.examples.MasterSlaveStateModelFactory;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -505,4 +508,70 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
.getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
2);
}
+
+ @Test
+ public void testAddCloudConfig() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+
+ CloudConfig.Builder builder = new CloudConfig.Builder();
+ builder.setCloudEnabled(true);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL");
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+
+ admin.addCloudConfig(clusterName, cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+
+ @Test
+ public void testRemoveCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+
+ CloudConfig.Builder builder = new CloudConfig.Builder();
+ builder.setCloudEnabled(true);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL");
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+
+ admin.addCloudConfig(clusterName, cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+
+ // Remove Cloud Config and make sure it has been removed from Zookeeper
+ admin.removeCloudConfig(clusterName);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNull(cloudConfigFromZk);
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index 6cb7790..e06c902 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -30,6 +30,7 @@ import org.apache.helix.HelixManager;
import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
import org.apache.helix.ZNRecord;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ConstraintItem;
@@ -312,6 +313,16 @@ public class MockHelixAdmin implements HelixAdmin {
}
+ @Override
+ public void addCloudConfig(String clusterName, CloudConfig cloudConfig) {
+
+ }
+
+ @Override
+ public void removeCloudConfig(String clusterName) {
+
+ }
+
@Override public List<String> getStateModelDefs(String clusterName) {
return null;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
index ee83011..01945de 100644
--- a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
@@ -57,20 +57,22 @@ public class TestCloudConfig extends ZkUnitTestBase {
}
@Test(dependsOnMethods = "testCloudConfigNull")
- public void testCloudConfig() {
+ public void testCloudConfig() throws Exception {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
// Create dummy CloudConfig object
- CloudConfig cloudConfig = new CloudConfig(clusterName);
- cloudConfig.setCloudEnabled(true);
- cloudConfig.setCloudProvider(CloudProvider.AZURE);
- cloudConfig.setCloudID("TestID");
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
List<String> infoURL = new ArrayList<String>();
infoURL.add("TestURL");
- cloudConfig.setCloudInfoSource(infoURL);
- cloudConfig.setCloudInfoFProcessorName("TestProcessor");
+ cloudConfigBuilder.setCloudInfoSources(infoURL);
+ cloudConfigBuilder.setCloudInfoProcessorName("TestProcessor");
+
+ CloudConfig cloudConfig = cloudConfigBuilder.build();
// Write the CloudConfig to Zookeeper
ZKHelixDataAccessor accessor =
@@ -84,8 +86,8 @@ public class TestCloudConfig extends ZkUnitTestBase {
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+
Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1);
- Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL");
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
}
@@ -93,7 +95,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
public void testUnverifiedCloudConfigBuilder() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
// Verify will fail because cloudID has net been defined.
CloudConfig cloudConfig = builder.build();
@@ -103,7 +105,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
public void testUnverifiedCloudConfigBuilderEmptySources() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -115,9 +117,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
@Test(expectedExceptions = HelixException.class)
public void testUnverifiedCloudConfigBuilderWithoutProcessor() {
- String className = getShortClassName();
- String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -132,7 +132,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -142,8 +142,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
// Check builder getter methods
Assert.assertTrue(builder.getCloudEnabled());
- Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED);
- Assert.assertEquals(builder.getClusterName(), clusterName);
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
Assert.assertEquals(builder.getCloudID(), "TestID");
List<String> listUrlFromBuilder = builder.getCloudInfoSources();
Assert.assertEquals(listUrlFromBuilder.size(), 2);
@@ -175,15 +174,14 @@ public class TestCloudConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.AZURE);
builder.setCloudID("TestID");
- builder.setCloudInfoProcessorName("TestProcessor");
// Check builder getter methods
Assert.assertTrue(builder.getCloudEnabled());
- Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE);
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE.name());
CloudConfig cloudConfig = builder.build();
@@ -198,7 +196,29 @@ public class TestCloudConfig extends ZkUnitTestBase {
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
- // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null.
+ // Since user does not set the CloudInfoProcessorName, this field will be null.
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+
+ // Checking the set method in CloudConfig
+ cloudConfig.setCloudEnabled(false);
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
+
+ cloudConfig.setCloudEnabled(true);
+ cloudConfig.setCloudID("TestID2");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL0");
+ sourceList.add("TestURL1");
+ cloudConfig.setCloudInfoSource(sourceList);
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID2");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromZk.get(1), "TestURL1");
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 84d5f51..6a6d48d 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -19,9 +19,12 @@ package org.apache.helix.tools;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.List;
import org.apache.helix.BaseDataAccessor;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey;
@@ -30,15 +33,15 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.AssertJUnit;
import org.testng.annotations.AfterClass;
@@ -47,8 +50,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class TestClusterSetup extends ZkUnitTestBase {
- private static Logger LOG = LoggerFactory.getLogger(TestClusterSetup.class);
-
protected static final String CLUSTER_NAME = "TestClusterSetup";
protected static final String TEST_DB = "TestDB";
protected static final String INSTANCE_PREFIX = "instance_";
@@ -435,4 +436,83 @@ public class TestClusterSetup extends ZkUnitTestBase {
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testAddClusterWithInvalidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+
+ // Since setCloudInfoProcessorName is missing, this add cluster call will throw an exception
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig")
+ public void testAddClusterWithValidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessorName");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
+
+
+ @Test(dependsOnMethods = "testAddClusterWithValidCloudConfig")
+ public void testAddClusterAzureProvider() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestID");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+
+ // Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
+ Assert.assertNull(listUrlFromZk);
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
}
[helix] 04/18: Add REST API for Cluster Creation with CloudConfig
(#675)
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch helix-0.9.x
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ffe8eea57d1e0f4da0eb1753497576b42feb22ce
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Jan 20 01:16:45 2020 -0800
Add REST API for Cluster Creation with CloudConfig (#675)
This commit contains the relevant REST APIs for cluster creation.
This change allows user to create cluster with CloudConfig.
Multiple tests have been added in order to check the functionality of REST calls.
---
.../server/resources/helix/ClusterAccessor.java | 146 +++++++++-
.../helix/rest/server/TestClusterAccessor.java | 302 ++++++++++++++++++++-
2 files changed, 440 insertions(+), 8 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index edfc847..f3175e4 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -46,6 +46,7 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.ZNRecord;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.manager.zk.client.HelixZkClient;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.HelixConfigScope;
@@ -134,17 +135,35 @@ public class ClusterAccessor extends AbstractHelixResource {
@PUT
@Path("{clusterId}")
public Response createCluster(@PathParam("clusterId") String clusterId,
- @DefaultValue("false") @QueryParam("recreate") String recreate) {
+ @DefaultValue("false") @QueryParam("recreate") String recreate,
+ @DefaultValue("false") @QueryParam("addCloudConfig") String addCloudConfig,
+ String content) {
+
boolean recreateIfExists = Boolean.valueOf(recreate);
+ boolean cloudConfigIncluded = Boolean.valueOf(addCloudConfig);
+
+
ClusterSetup clusterSetup = getClusterSetup();
- try {
- clusterSetup.addCluster(clusterId, recreateIfExists);
- } catch (Exception ex) {
- _logger.error("Failed to create cluster " + clusterId + ", exception: " + ex);
- return serverError(ex);
- }
+ CloudConfig cloudConfig = null;
+ if (cloudConfigIncluded) {
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+ try {
+ cloudConfig = new CloudConfig.Builder(record).build();
+ clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
+ } catch (Exception ex) {
+ _logger.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ }
+ }
+ clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
return created();
}
@@ -516,6 +535,119 @@ public class ClusterAccessor extends AbstractHelixResource {
return ZKUtil.isClusterSetup(cluster, zkClient);
}
+ @PUT
+ @Path("{clusterId}/cloudconfig")
+ public Response addCloudConfig(@PathParam("clusterId") String clusterId, String content) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound("Cluster is not properly setup!");
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+
+ try {
+ CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
+ admin.addCloudConfig(clusterId, cloudConfig);
+ } catch (HelixException ex) {
+ _logger.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ } catch (Exception ex) {
+ _logger.error("Cannot add CloudConfig to cluster: " + clusterId, ex);
+ return serverError(ex);
+ }
+
+ return OK();
+ }
+
+ @GET
+ @Path("{clusterId}/cloudconfig")
+ public Response getCloudConfig(@PathParam("clusterId") String clusterId) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound();
+ }
+
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
+ CloudConfig cloudConfig = configAccessor.getCloudConfig(clusterId);
+
+ if (cloudConfig != null) {
+ return JSONRepresentation(cloudConfig.getRecord());
+ }
+
+ return notFound();
+ }
+
+ @POST
+ @Path("{clusterId}/cloudconfig")
+ public Response updateCloudConfig(@PathParam("clusterId") String clusterId,
+ @QueryParam("command") String commandStr, String content) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound();
+ }
+
+ // Here to update cloud config
+ Command command;
+ if (commandStr == null || commandStr.isEmpty()) {
+ command = Command.update; // Default behavior
+ } else {
+ try {
+ command = getCommand(commandStr);
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ _logger.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+ try {
+ switch (command) {
+ case delete: {
+ admin.removeCloudConfig(clusterId);
+ }
+ break;
+ case update: {
+ try {
+ CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
+ admin.removeCloudConfig(clusterId);
+ admin.addCloudConfig(clusterId, cloudConfig);
+ } catch (HelixException ex) {
+ _logger.error("Error in updating a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ } catch (Exception ex) {
+ _logger.error("Cannot update CloudConfig for cluster: " + clusterId, ex);
+ return serverError(ex);
+ }
+ }
+ break;
+ default:
+ return badRequest("Unsupported command " + commandStr);
+ }
+ } catch (Exception ex) {
+ _logger.error("Failed to " + command + " cloud config, cluster " + clusterId + " new config: "
+ + content + ", Exception: " + ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
/**
* Reads HISTORY ZNode from the metadata store and generates a Map object that contains the
* pertinent history entries depending on the history type.
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 32ea5ed..2df28bd 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -31,7 +31,9 @@ import java.util.Set;
import javax.ws.rs.client.Entity;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-
+import com.google.common.collect.ImmutableMap;
+import com.sun.research.ws.wadl.HTTPMethods;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
@@ -47,6 +49,8 @@ import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
+import org.apache.helix.model.CloudConfig;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.rest.common.HelixRestNamespace;
import org.apache.helix.rest.server.auditlog.AuditLog;
import org.apache.helix.rest.server.resources.AbstractResource;
@@ -565,6 +569,302 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+
+ @Test(dependsOnMethods = "testActivateSuperCluster")
+ public void testAddClusterWithCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithCloudConfig")
+ public void testAddClusterWithInvalidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider has not been defined. Result of this rest call will be BAD_REQUEST.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig")
+ public void testAddClusterWithInvalidCustomizedCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider is customized. CLOUD_INFO_PROCESSOR_NAME and CLOUD_INFO_SOURCE fields are
+ // required.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCustomizedCloudConfig")
+ public void testAddClusterWithValidCustomizedCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), sourceList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider is customized. CLOUD_INFO_PROCESSOR_NAME and CLOUD_INFO_SOURCE fields are
+ // required.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithValidCustomizedCloudConfig")
+ public void testAddClusterWithCloudConfigDisabledCloud() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), false);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
+
+
+ @Test(dependsOnMethods = "testAddClusterWithCloudConfigDisabledCloud")
+ public void testAddCloudConfigNonExistedCluster() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessor");
+
+ // Not found since the cluster is not setup yet.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.NOT_FOUND.getStatusCode());
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCloudConfigNonExistedCluster")
+ public void testAddCloudConfig() throws Exception {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+
+ // Bad request since Processor has not been defined.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ // Now response should be OK since all fields are set
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+
+ // Now test the getCloudConfig method.
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CloudConfig cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ CloudConfig cloudConfigZk = _configAccessor.getCloudConfig("TestCloud");
+
+ // Check that the CloudConfig from Zk and REST get method are equal
+ Assert.assertEquals(cloudConfigRest, cloudConfigZk);
+
+ // Check the fields individually
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromRest = cloudConfigRest.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromRest.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigRest.getCloudInfoProcessorName(), "TestProcessorName");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCloudConfig")
+ public void testDeleteCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", AbstractResource.Command.delete.name());
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and make sure it has been removed
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ Assert.assertNull(cloudConfigFromZk);
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testDeleteCloudConfig")
+ public void testUpdateCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ // Fist add CloudConfig to the cluster
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Now get the Cloud Config and make sure the information is correct
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CloudConfig cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Now put new information in the ZNRecord
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudIdNew");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", AbstractResource.Command.update.name());
+
+ post(urlBase, map1,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Now get the Cloud Config and make sure the information has been updated
+ body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudIdNew");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromRest = cloudConfigRest.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromRest.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigRest.getCloudInfoProcessorName(), "TestProcessorName");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+
private ClusterConfig getClusterConfigFromRest(String cluster) throws IOException {
String body = get("clusters/" + cluster + "/configs", null, Response.Status.OK.getStatusCode(), true);