You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2020/04/07 01:20:29 UTC
[helix] branch master updated (184a50a -> ec2f3d9)
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git.
from 184a50a Fix the scheduling decision for multiple currentStates (#923)
new ba0f4e5 Add CloudConfig code
new 64ca238 add Helix cloud interface and implementation skeleton methods
new 9f6cc4b Add java API to create cluster with CloudConfig
new 3bbeb31 Add REST API for Cluster Creation with CloudConfig (#675)
new 5dac1cc Add Helix properties factory and class (#653)
new 079483b Implement Azure cloud instance information processor (#698)
new c2f0dc3 Modify participant manager to add cluster auto registration logic (#695)
new a32f387 add one more test for auto registration (#806)
new 912e794 Change the cluster creation logic (#872)
new 7f640e4 Add construction of domain in Helix participant logic (#876)
new ab3c4da Change the REST call for delete CloudConfig (#882)
new ec2f3d9 Add REST and JAVA API to modify existing cloudconfig (#898)
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
helix-core/pom.xml | 5 +
.../main/java/org/apache/helix/ConfigAccessor.java | 65 +++-
.../src/main/java/org/apache/helix/HelixAdmin.java | 15 +-
.../java/org/apache/helix/HelixCloudProperty.java | 183 +++++++++++
.../java/org/apache/helix/HelixManagerFactory.java | 2 +-
.../org/apache/helix/HelixManagerProperty.java | 74 +++++
.../org/apache/helix/HelixPropertyFactory.java | 79 +++++
.../main/java/org/apache/helix/PropertyKey.java | 11 +
.../java/org/apache/helix/SystemPropertyKeys.java | 8 +
.../cloud/CloudInstanceInformation.java} | 24 +-
.../cloud/CloudInstanceInformationProcessor.java} | 24 +-
.../cloud/azure/AzureCloudInstanceInformation.java | 72 ++++
.../AzureCloudInstanceInformationProcessor.java | 160 +++++++++
.../apache/helix/cloud/azure/AzureConstants.java | 6 +
.../constants/CloudProvider.java} | 7 +-
.../helix/manager/zk/ParticipantManager.java | 111 +++++--
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++
.../apache/helix/manager/zk/ZKHelixManager.java | 18 +-
.../java/org/apache/helix/model/CloudConfig.java | 265 +++++++++++++++
.../org/apache/helix/model/HelixConfigScope.java | 6 +-
.../model/builder/HelixConfigScopeBuilder.java | 3 +
.../java/org/apache/helix/tools/ClusterSetup.java | 28 +-
.../main/java/org/apache/helix/util/HelixUtil.java | 19 ++
...nt-weight.properties => azure-cloud.properties} | 13 +-
...version.properties => helix-manager.properties} | 5 +-
.../java/org/apache/helix/TestConfigAccessor.java | 99 +++++-
.../org/apache/helix/cloud/MockHttpClient.java | 53 +++
...TestAzureCloudInstanceInformationProcessor.java | 69 ++++
.../manager/MockParticipantManager.java | 10 +-
.../paticipant/TestInstanceAutoJoin.java | 62 +++-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 70 ++++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 11 +
.../apache/helix/model/cloud/TestCloudConfig.java | 224 +++++++++++++
.../org/apache/helix/tools/TestClusterSetup.java | 100 +++++-
helix-core/src/test/resources/AzureResponse.json | 104 ++++++
helix-rest/pom.xml | 5 -
.../server/resources/helix/ClusterAccessor.java | 146 ++++++++-
.../helix/rest/server/TestClusterAccessor.java | 362 +++++++++++++++++++++
38 files changed, 2456 insertions(+), 90 deletions(-)
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
create mode 100644 helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
copy helix-core/src/main/java/org/apache/helix/{monitoring/metrics/implementation/RebalanceCounter.java => api/cloud/CloudInstanceInformation.java} (65%)
copy helix-core/src/main/java/org/apache/helix/{monitoring/metrics/implementation/RebalanceCounter.java => api/cloud/CloudInstanceInformationProcessor.java} (59%)
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
create mode 100644 helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
copy helix-core/src/main/java/org/apache/helix/{controller/pipeline/StageContext.java => cloud/constants/CloudProvider.java} (89%)
create mode 100644 helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
copy helix-core/src/main/resources/{soft-constraint-weight.properties => azure-cloud.properties} (72%)
copy helix-core/src/main/resources/{cluster-manager-version.properties => helix-manager.properties} (88%)
create mode 100644 helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
create mode 100644 helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
create mode 100644 helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
create mode 100644 helix-core/src/test/resources/AzureResponse.json
[helix] 05/12: Add Helix properties factory and class (#653)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 5dac1cc423b4db15d5597e76a28d5c74c782b1a5
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Jan 21 14:05:50 2020 -0800
Add Helix properties factory and class (#653)
- Add Helix property singleton factory. The factory returns Helix property with default values, and clients may override these values;
- Add Helix manager property. It further holds Helix cloud properties and some other properties specific to the manager, defined in helix-manager.properties file.
- Add Helix cloud property. It holds cloud related properties, which comes from cloud config and client specified file, e.g., for Azure, the file is azure-cloud.properties.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 2 -
.../java/org/apache/helix/HelixCloudProperty.java | 183 +++++++++++++++++++++
.../java/org/apache/helix/HelixManagerFactory.java | 2 +-
.../org/apache/helix/HelixManagerProperty.java | 74 +++++++++
.../org/apache/helix/HelixPropertyFactory.java | 79 +++++++++
.../java/org/apache/helix/SystemPropertyKeys.java | 8 +
.../cloud/azure/AzureCloudInstanceInformation.java | 18 +-
.../helix/manager/zk/ParticipantManager.java | 12 ++
.../apache/helix/manager/zk/ZKHelixManager.java | 18 +-
.../java/org/apache/helix/model/CloudConfig.java | 36 +++-
.../src/main/resources/azure-cloud.properties | 25 +++
.../src/main/resources/helix-manager.properties | 24 +++
12 files changed, 455 insertions(+), 26 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index e09ae5b..c0afde4 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -610,9 +610,7 @@ public class ConfigAccessor {
/**
* Get RestConfig of the given cluster.
- *
* @param clusterName The cluster
- *
* @return The instance of {@link RESTConfig}
*/
public RESTConfig getRESTConfig(String clusterName) {
diff --git a/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
new file mode 100644
index 0000000..554c595
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixCloudProperty.java
@@ -0,0 +1,183 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hold helix cloud properties read from CloudConfig and user defined files. Clients may override
+ * the fields from their application.
+ */
+public class HelixCloudProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixCloudProperty.class.getName());
+ private static final String AZURE_CLOUD_PROPERTY_FILE = SystemPropertyKeys.AZURE_CLOUD_PROPERTIES;
+ private static final String CLOUD_INFO_SOURCE = "cloud_info_source";
+ private static final String CLOUD_INFO_PROCESSFOR_NAME = "cloud_info_processor_name";
+ private static final String CLOUD_MAX_RETRY = "cloud_max_retry";
+ private static final String CONNECTION_TIMEOUT_MS = "connection_timeout_ms";
+ private static final String REQUEST_TIMEOUT_MS = "request_timeout_ms";
+
+ // Denote whether the instance is considered as in a cloud environment.
+ private boolean _isCloudEnabled;
+
+ // Unique id of the cloud environment where the instance is in.
+ private String _cloudId;
+
+ // Cloud environment provider, e.g. Azure, AWS, GCP, etc.
+ private String _cloudProvider;
+
+ // The sources where the cloud instance information can be retrieved from.
+ private List<String> _cloudInfoSources;
+
+ // The name of the function that will fetch and parse cloud instance information.
+ private String _cloudInfoProcessorName;
+
+ // Http max retry times when querying the cloud instance information from cloud environment.
+ private int _cloudMaxRetry;
+
+ // Http connection time when querying the cloud instance information from cloud environment.
+ private long _cloudConnectionTimeout;
+
+ // Http request timeout when querying the cloud instance information from cloud environment.
+ private long _cloudRequestTimeout;
+
+ // Other customized properties that may be used.
+ private Properties _customizedCloudProperties = new Properties();
+
+ /**
+ * Initialize Helix Cloud Property based on the provider
+ * @param
+ */
+ public HelixCloudProperty(CloudConfig cloudConfig) {
+ setCloudEndabled(cloudConfig.isCloudEnabled());
+ if (cloudConfig.isCloudEnabled()) {
+ setCloudId(cloudConfig.getCloudID());
+ setCloudProvider(cloudConfig.getCloudProvider());
+ switch (CloudProvider.valueOf(cloudConfig.getCloudProvider())) {
+ case AZURE:
+ Properties azureProperties = new Properties();
+ try {
+ InputStream stream = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(AZURE_CLOUD_PROPERTY_FILE);
+ azureProperties.load(stream);
+ } catch (IOException e) {
+ String errMsg =
+ "failed to open Helix Azure cloud properties file: " + AZURE_CLOUD_PROPERTY_FILE;
+ throw new IllegalArgumentException(errMsg, e);
+ }
+ LOG.info("Successfully loaded Helix Azure cloud properties: {}", azureProperties);
+ setCloudInfoSources(
+ Collections.singletonList(azureProperties.getProperty(CLOUD_INFO_SOURCE)));
+ setCloudInfoProcessorName(azureProperties.getProperty(CLOUD_INFO_PROCESSFOR_NAME));
+ setCloudMaxRetry(Integer.valueOf(azureProperties.getProperty(CLOUD_MAX_RETRY)));
+ setCloudConnectionTimeout(Long.valueOf(azureProperties.getProperty(CONNECTION_TIMEOUT_MS)));
+ setCloudRequestTimeout(Long.valueOf(azureProperties.getProperty(REQUEST_TIMEOUT_MS)));
+ break;
+ case CUSTOMIZED:
+ setCloudInfoSources(cloudConfig.getCloudInfoSources());
+ setCloudInfoProcessorName(cloudConfig.getCloudInfoProcessorName());
+ break;
+ default:
+ throw new HelixException(
+ String.format("Unsupported cloud provider: %s", cloudConfig.getCloudProvider()));
+ }
+ }
+ }
+
+ public boolean getCloudEnabled() {
+ return _isCloudEnabled;
+ }
+
+ public String getCloudId() {
+ return _cloudId;
+ }
+
+ public String getCloudProvider() {
+ return _cloudProvider;
+ }
+
+ public List<String> getCloudInfoSources() {
+ return _cloudInfoSources;
+ }
+
+ public String getCloudInfoProcessorName() {
+ return _cloudInfoProcessorName;
+ }
+
+ public int getCloudMaxRetry() {
+ return _cloudMaxRetry;
+ }
+
+ public long getCloudConnectionTimeout() {
+ return _cloudConnectionTimeout;
+ }
+
+ public long getCloudRequestTimeout() {
+ return _cloudRequestTimeout;
+ }
+
+ public Properties getCustomizedCloudProperties() {
+ return _customizedCloudProperties;
+ }
+
+ public void setCloudEndabled(boolean isCloudEnabled) {
+ _isCloudEnabled = isCloudEnabled;
+ }
+
+ public void setCloudId(String cloudId) {
+ _cloudId = cloudId;
+ }
+
+ public void setCloudProvider(String cloudProvider) {
+ _cloudProvider = cloudProvider;
+ }
+
+ public void setCloudInfoSources(List<String> sources) {
+ _cloudInfoSources = sources;
+ }
+
+ public void setCloudInfoProcessorName(String cloudInfoProcessorName) {
+ _cloudInfoProcessorName = cloudInfoProcessorName;
+ }
+
+ public void setCloudMaxRetry(int cloudMaxRetry) {
+ _cloudMaxRetry = cloudMaxRetry;
+ }
+
+ public void setCloudConnectionTimeout(long cloudConnectionTimeout) {
+ _cloudConnectionTimeout = cloudConnectionTimeout;
+ }
+
+ public void setCloudRequestTimeout(long cloudRequestTimeout) {
+ _cloudRequestTimeout = cloudRequestTimeout;
+ }
+
+ public void setCustomizedCloudProperties(Properties customizedCloudProperties) {
+ _customizedCloudProperties.putAll(customizedCloudProperties);
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java b/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
index 7d8038d..ee53c1d 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixManagerFactory.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
* Obtain one of a set of Helix cluster managers, organized by the backing system.
*/
public final class HelixManagerFactory {
- private static final Logger logger = LoggerFactory.getLogger(HelixManagerFactory.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HelixManagerFactory.class);
/**
* Construct a zk-based cluster manager that enforces all types (PARTICIPANT, CONTROLLER, and
diff --git a/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java b/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
new file mode 100644
index 0000000..2ec26a7
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixManagerProperty.java
@@ -0,0 +1,74 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Properties;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hold Helix manager properties. The manager properties further hold Helix cloud properties
+ * and some other properties specific for the manager.
+ */
+public class HelixManagerProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixManagerProperty.class.getName());
+ private String _version;
+ private long _healthReportLatency;
+ private HelixCloudProperty _helixCloudProperty;
+
+ /**
+ * Initialize Helix manager property with default value
+ * @param helixManagerProperties helix manager related properties input as a map
+ * @param cloudConfig cloudConfig read from Zookeeper
+ */
+ public HelixManagerProperty(Properties helixManagerProperties, CloudConfig cloudConfig) {
+ _helixCloudProperty = new HelixCloudProperty(cloudConfig);
+ setVersion(helixManagerProperties.getProperty(SystemPropertyKeys.HELIX_MANAGER_VERSION));
+ setHealthReportLatency(
+ helixManagerProperties.getProperty(SystemPropertyKeys.PARTICIPANT_HEALTH_REPORT_LATENCY));
+ }
+
+ public HelixCloudProperty getHelixCloudProperty() {
+ return _helixCloudProperty;
+ }
+
+ public String getVersion() {
+ return _version;
+ }
+
+ public long getHealthReportLatency() {
+ return _healthReportLatency;
+ }
+
+ public void setHelixCloudProperty(HelixCloudProperty helixCloudProperty) {
+ _helixCloudProperty = helixCloudProperty;
+ }
+
+ public void setVersion(String version) {
+ _version = version;
+ }
+
+ public void setHealthReportLatency(String latency) {
+ _healthReportLatency = Long.valueOf(latency);
+ }
+
+ // TODO: migrate all other participant related properties to this file.
+}
diff --git a/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
new file mode 100644
index 0000000..fa394a2
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/HelixPropertyFactory.java
@@ -0,0 +1,79 @@
+package org.apache.helix;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import org.apache.helix.model.CloudConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Singleton factory that builds different types of Helix property, e.g. Helix manager property.
+ */
+public final class HelixPropertyFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(HelixPropertyFactory.class);
+ private static final String HELIX_PARTICIPANT_PROPERTY_FILE =
+ SystemPropertyKeys.HELIX_MANAGER_PROPERTIES;
+
+ private static class SingletonHelper {
+ private static final HelixPropertyFactory INSTANCE = new HelixPropertyFactory();
+ }
+
+ public static HelixPropertyFactory getInstance() {
+ return SingletonHelper.INSTANCE;
+ }
+
+ /**
+ * Retrieve Helix manager property. It returns the property object with default values.
+ * Clients may override these values.
+ */
+ public HelixManagerProperty getHelixManagerProperty(String zkAddress, String clusterName) {
+ ConfigAccessor configAccessor = new ConfigAccessor(zkAddress);
+ CloudConfig cloudConfig;
+ // The try-catch logic is for backward compatibility reason only. Even if the cluster is not set
+ // up yet, constructing a new ZKHelixManager should not throw an exception
+ try {
+ cloudConfig =
+ configAccessor.getCloudConfig(clusterName) == null ? buildEmptyCloudConfig(clusterName)
+ : configAccessor.getCloudConfig(clusterName);
+ } catch (HelixException e) {
+ cloudConfig = buildEmptyCloudConfig(clusterName);
+ }
+ Properties properties = new Properties();
+ try {
+ InputStream stream = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(HELIX_PARTICIPANT_PROPERTY_FILE);
+ properties.load(stream);
+ } catch (IOException e) {
+ String errMsg = String.format("failed to open Helix participant properties file: %s",
+ HELIX_PARTICIPANT_PROPERTY_FILE);
+ throw new IllegalArgumentException(errMsg, e);
+ }
+ LOG.info("HelixPropertyFactory successfully loaded helix participant properties: {}",
+ properties);
+ return new HelixManagerProperty(properties, cloudConfig);
+ }
+
+ public static CloudConfig buildEmptyCloudConfig(String clusterName) {
+ return new CloudConfig.Builder().setCloudEnabled(false).build();
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java b/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
index 2d824cb..d30a2de 100644
--- a/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
+++ b/helix-core/src/main/java/org/apache/helix/SystemPropertyKeys.java
@@ -20,6 +20,14 @@ package org.apache.helix;
*/
public class SystemPropertyKeys {
+ // Used to compose default values in HelixManagerProperty
+ public static final String HELIX_MANAGER_PROPERTIES = "helix-manager.properties";
+
+ public static final String HELIX_MANAGER_VERSION = "clustermanager.version";
+
+ // Used to compose default values in HelixCloudProperty when cloud provider is Azure
+ public static final String AZURE_CLOUD_PROPERTIES = "azure-cloud.properties";
+
// Task Driver
public static final String TASK_CONFIG_LIMITATION = "helixTask.configsLimitation";
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
index 511f3e3..f7fd657 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -48,24 +48,18 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
}
- /**
- * Default constructor
- */
- public Builder() {
- }
-
- public Builder setInstanceName(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), v);
+ public Builder setInstanceName(String name) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), name);
return this;
}
- public Builder setFaultDomain(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), v);
+ public Builder setFaultDomain(String faultDomain) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), faultDomain);
return this;
}
- public Builder setInstanceSetName(String v) {
- _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), v);
+ public Builder setInstanceSetName(String instanceSetName) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), instanceSetName);
return this;
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 411d937..a9b5bde 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -31,6 +31,7 @@ import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
import org.apache.helix.PreConnectCallback;
@@ -76,14 +77,24 @@ public class ParticipantManager {
final StateMachineEngine _stateMachineEngine;
final LiveInstanceInfoProvider _liveInstanceInfoProvider;
final List<PreConnectCallback> _preConnectCallbacks;
+ final HelixManagerProperty _helixManagerProperty;
// zk session id should be immutable after participant manager is created. This is to avoid
// session race condition when handling new session for the participant.
private final String _sessionId;
+ @Deprecated
public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout,
LiveInstanceInfoProvider liveInstanceInfoProvider, List<PreConnectCallback> preConnectCallbacks,
final String sessionId) {
+ this(manager, zkclient, sessionTimeout, liveInstanceInfoProvider, preConnectCallbacks,
+ sessionId, null);
+ }
+
+ public ParticipantManager(HelixManager manager, HelixZkClient zkclient, int sessionTimeout,
+ LiveInstanceInfoProvider liveInstanceInfoProvider,
+ List<PreConnectCallback> preConnectCallbacks, final String sessionId,
+ HelixManagerProperty helixManagerProperty) {
_zkclient = zkclient;
_manager = manager;
_clusterName = manager.getClusterName();
@@ -99,6 +110,7 @@ public class ParticipantManager {
_stateMachineEngine = manager.getStateMachineEngine();
_liveInstanceInfoProvider = liveInstanceInfoProvider;
_preConnectCallbacks = preConnectCallbacks;
+ _helixManagerProperty = helixManagerProperty;
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
index 532ef44..a0fb609 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixManager.java
@@ -41,6 +41,8 @@ import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerProperties;
+import org.apache.helix.HelixManagerProperty;
+import org.apache.helix.HelixPropertyFactory;
import org.apache.helix.HelixTimerTask;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
@@ -109,6 +111,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
private final List<PreConnectCallback> _preConnectCallbacks;
protected final List<CallbackHandler> _handlers;
private final HelixManagerProperties _properties;
+ private final HelixManagerProperty _helixManagerProperty;
private final HelixManagerStateListener _stateListener;
/**
@@ -204,9 +207,16 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
public ZKHelixManager(String clusterName, String instanceName, InstanceType instanceType,
String zkAddress, HelixManagerStateListener stateListener) {
+ this(clusterName, instanceName, instanceType, zkAddress, stateListener,
+ HelixPropertyFactory.getInstance().getHelixManagerProperty(zkAddress, clusterName));
+ }
+
+ public ZKHelixManager(String clusterName, String instanceName, InstanceType instanceType,
+ String zkAddress, HelixManagerStateListener stateListener,
+ HelixManagerProperty helixManagerProperty) {
- LOG.info(
- "Create a zk-based cluster manager. zkSvr: " + zkAddress + ", clusterName: " + clusterName + ", instanceName: " + instanceName + ", type: " + instanceType);
+ LOG.info("Create a zk-based cluster manager. zkSvr: " + zkAddress + ", clusterName: "
+ + clusterName + ", instanceName: " + instanceName + ", type: " + instanceType);
_zkAddress = zkAddress;
_clusterName = clusterName;
@@ -246,6 +256,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
_stateListener = stateListener;
+ _helixManagerProperty = helixManagerProperty;
/**
* use system property if available
@@ -1247,8 +1258,7 @@ public class ZKHelixManager implements HelixManager, IZkStateListener {
}
_participantManager =
new ParticipantManager(this, _zkclient, _sessionTimeout, _liveInstanceInfoProvider,
- _preConnectCallbacks, sessionId);
-
+ _preConnectCallbacks, sessionId, _helixManagerProperty);
_participantManager.handleNewSession();
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index c1ffa61..0fd622b 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -73,6 +73,26 @@ public class CloudConfig extends HelixProperty {
_record.setMapFields(record.getMapFields());
}
+ /**
+ * Instantiate the config using each field individually.
+ * Users should use CloudConfig.Builder to create CloudConfig.
+ * @param cluster
+ * @param enabled
+ * @param cloudID
+ */
+ public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
+ List<String> cloudInfoSource, String cloudProcessorName) {
+ super(cluster);
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ if (enabled == true) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
+ }
+ }
+ }
/**
* Enable/Disable the CLOUD_ENABLED field.
@@ -228,14 +248,16 @@ public class CloudConfig extends HelixProperty {
}
private void validate() {
- if (this.getCloudProvider() == null) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
- } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
- if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
- || this.getCloudInfoSources().size() == 0) {
+ if (this.getCloudEnabled()) {
+ if (this.getCloudProvider() == null) {
throw new HelixException(
- "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
+ if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
+ || this.getCloudInfoSources().size() == 0) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ }
}
}
}
diff --git a/helix-core/src/main/resources/azure-cloud.properties b/helix-core/src/main/resources/azure-cloud.properties
new file mode 100644
index 0000000..b5321fc
--- /dev/null
+++ b/helix-core/src/main/resources/azure-cloud.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This is the globally fixed value for Azure Metadata Instance Service endpoint
+cloud_info_source=http://169.254.169.254/metadata/instance?api-version=2019-06-04
+cloud_info_processor_name=AzureCloudInstanceInformationProcessor
+cloud_max_retry=5
+connection_timeout_ms=5000
+request_timeout_ms=5000
\ No newline at end of file
diff --git a/helix-core/src/main/resources/helix-manager.properties b/helix-core/src/main/resources/helix-manager.properties
new file mode 100644
index 0000000..eafcb3a
--- /dev/null
+++ b/helix-core/src/main/resources/helix-manager.properties
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+clustermanager.version=${project.version}
+
+minimum_supported_version.participant=0.4
+
+helixmanager.participantHealthReport.reportLatency=60000
\ No newline at end of file
[helix] 12/12: Add REST and JAVA API to modify existing cloudconfig
(#898)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ec2f3d983acfc2ae4605a8b9452ea6c335a95b80
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Mar 17 13:07:46 2020 -0700
Add REST and JAVA API to modify existing cloudconfig (#898)
The new APIs are added to update/delete cloudconfig entries.
Tests have been modified accordingly.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 42 +++++++++++++++-
.../AzureCloudInstanceInformationProcessor.java | 8 ++--
.../java/org/apache/helix/model/CloudConfig.java | 2 +-
.../java/org/apache/helix/TestConfigAccessor.java | 56 ++++++++++++++++++++--
...TestAzureCloudInstanceInformationProcessor.java | 8 ++--
.../server/resources/helix/ClusterAccessor.java | 13 +++--
.../helix/rest/server/TestClusterAccessor.java | 50 ++++++++++++++++++-
7 files changed, 157 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index c0afde4..8ae7fd5 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -605,7 +605,47 @@ public class ConfigAccessor {
return null;
}
- return new CloudConfig.Builder(record).build();
+ return new CloudConfig(record);
+ }
+
+ /**
+ * Delete cloud config fields (not the whole config)
+ * @param clusterName
+ * @param cloudConfig
+ */
+ public void deleteCloudConfigFields(String clusterName, CloudConfig cloudConfig) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("fail to delete cloud config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ remove(scope, cloudConfig.getRecord());
+ }
+
+ /**
+ * Update cloud config
+ * @param clusterName
+ * @param cloudConfig
+ */
+ public void updateCloudConfig(String clusterName, CloudConfig cloudConfig) {
+ updateCloudConfig(clusterName, cloudConfig, false);
+ }
+
+ private void updateCloudConfig(String clusterName, CloudConfig cloudConfig, boolean overwrite) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("Fail to update cloud config. cluster: " + clusterName + " is NOT setup.");
+ }
+
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ String zkPath = scope.getZkPath();
+
+ if (overwrite) {
+ ZKUtil.createOrReplace(_zkClient, zkPath, cloudConfig.getRecord(), true);
+ } else {
+ ZKUtil.createOrUpdate(_zkClient, zkPath, cloudConfig.getRecord(), true, true);
+ }
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 464ff55..f0f75f7 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -145,15 +145,15 @@ public class AzureCloudInstanceInformationProcessor
String azureTopology = AzureConstants.AZURE_TOPOLOGY;
String[] parts = azureTopology.trim().split("/");
//The hostname will be filled in by each participant
- String domain = parts[0] + "=" + platformFaultDomain + "," + parts[1] + "=";
+ String domain = parts[1] + "=" + platformFaultDomain + "," + parts[2] + "=";
AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
- builder.setInstanceName(vmName).setFaultDomain(domain)
- .setInstanceSetName(vmssName);
+ builder.setInstanceName(vmName).setFaultDomain(domain).setInstanceSetName(vmssName);
azureCloudInstanceInformation = builder.build();
}
} catch (IOException e) {
- throw new HelixException(String.format("Error in parsing cloud instance information: {}", response, e));
+ throw new HelixException(
+ String.format("Error in parsing cloud instance information: {}", response, e));
}
return azureCloudInstanceInformation;
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index 0fd622b..0c6062d 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -66,7 +66,7 @@ public class CloudConfig extends HelixProperty {
* The constructor from the ZNRecord.
* @param record
*/
- private CloudConfig(ZNRecord record) {
+ public CloudConfig(ZNRecord record) {
super(CLOUD_CONFIG_KW);
_record.setSimpleFields(record.getSimpleFields());
_record.setListFields(record.getListFields());
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index 4452c23..e44585f 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -300,7 +300,7 @@ public class TestConfigAccessor extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -310,10 +310,10 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
// Change the processor name and check if processor name has been changed in Zookeeper.
- cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2");
- cloudConfigInit = cloudConfigInitBuilder.build();
- ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
- admin.addCloudConfig(clusterName, cloudConfigInit);
+ CloudConfig.Builder cloudConfigToUpdateBuilder = new CloudConfig.Builder();
+ cloudConfigToUpdateBuilder.setCloudInfoProcessorName("TestProcessor2");
+ CloudConfig cloudConfigToUpdate = cloudConfigToUpdateBuilder.build();
+ _configAccessor.updateCloudConfig(clusterName, cloudConfigToUpdate);
cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
@@ -323,4 +323,50 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
}
+
+ @Test
+ public void testDeleteCloudConfig() throws Exception {
+ ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR);
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestCloudID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE);
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Change the processor name and check if processor name has been changed in Zookeeper.
+ CloudConfig.Builder cloudConfigBuilderToDelete = new CloudConfig.Builder();
+ cloudConfigBuilderToDelete.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigBuilderToDelete.setCloudID("TestCloudID");
+ CloudConfig cloudConfigToDelete = cloudConfigBuilderToDelete.build();
+
+ _configAccessor.deleteCloudConfigFields(clusterName, cloudConfigToDelete);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertNull(cloudConfigFromZk.getCloudID());
+ listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
index 10ba42d..350c9a9 100644
--- a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -55,12 +55,12 @@ public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
// Verify the response from mock http client
AzureCloudInstanceInformation azureCloudInstanceInformation =
processor.parseCloudInstanceInformation(response);
- Assert.assertEquals(azureCloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()), "2");
Assert.assertEquals(
azureCloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()),
- "test-helix");
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()),
+ "faultDomain=2," + "hostname=");
+ Assert.assertEquals(azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()), "test-helix");
Assert.assertEquals(
azureCloudInstanceInformation
.get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 3966d7c..95fb534 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -733,6 +733,7 @@ public class ClusterAccessor extends AbstractHelixResource {
return notFound();
}
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
// Here to update cloud config
Command command;
if (commandStr == null || commandStr.isEmpty()) {
@@ -745,22 +746,24 @@ public class ClusterAccessor extends AbstractHelixResource {
}
}
- HelixAdmin admin = getHelixAdmin();
-
ZNRecord record;
+ CloudConfig cloudConfig;
try {
record = toZNRecord(content);
+ cloudConfig = new CloudConfig(record);
} catch (IOException e) {
LOG.error("Failed to deserialize user's input " + content + ", Exception: " + e);
return badRequest("Input is not a vaild ZNRecord!");
}
try {
switch (command) {
+ case delete: {
+ configAccessor.deleteCloudConfigFields(clusterId, cloudConfig);
+ }
+ break;
case update: {
try {
- CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
- admin.removeCloudConfig(clusterId);
- admin.addCloudConfig(clusterId, cloudConfig);
+ configAccessor.updateCloudConfig(clusterId, cloudConfig);
} catch (HelixException ex) {
LOG.error("Error in updating a CloudConfig to cluster: " + clusterId, ex);
return badRequest(ex.getMessage());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 78a7a2c..9e862a8 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -871,7 +871,7 @@ public class TestClusterAccessor extends AbstractTestClass {
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
-
+
ZNRecord record = new ZNRecord("testZnode");
record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
@@ -888,7 +888,6 @@ public class TestClusterAccessor extends AbstractTestClass {
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNotNull(cloudConfigFromZk);
String urlBase = "clusters/" + clusterName + "/cloudconfig/";
-
delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
@@ -899,7 +898,54 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+
@Test(dependsOnMethods = "testDeleteCloudConfig")
+ public void testPartialDeleteCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+
+ ZNRecord record = new ZNRecord(clusterName);
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), "TestProcessor");
+ _gSetupTool.addCluster(clusterName, true, new CloudConfig.Builder(record).build());
+
+ String urlBase = "clusters/" + clusterName +"/cloudconfig/";
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+ // Read CloudConfig from Zookeeper and make sure it has been created
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNotNull(cloudConfigFromZk);
+
+ record = new ZNRecord(clusterName);
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", Command.delete.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(), CloudProvider.AZURE.name());
+ post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and make sure it has been removed
+ _configAccessor = new ConfigAccessor(ZK_ADDR);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNull(cloudConfigFromZk.getCloudID());
+ Assert.assertNull(cloudConfigFromZk.getCloudProvider());
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(),"TestProcessor");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testPartialDeleteCloudConfig")
public void testUpdateCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
_gSetupTool.addCluster("TestCloud", true);
[helix] 07/12: Modify participant manager to add cluster auto
registration logic (#695)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit c2f0dc3e6007762d55ee31c713354ef3d21222c1
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Fri Feb 7 15:41:36 2020 -0800
Modify participant manager to add cluster auto registration logic (#695)
Modify participant logic to add logic for instance auto registration to a Helix cluster.
Auto registration differs with existing auto join in that auto registration will retrieve the fault domain information for the instance and populate it under instanceConfig. With auto registration on, user does not need to manually populate instance information in Zookeeper.
---
.../helix/manager/zk/ParticipantManager.java | 99 +++++++++++++++++-----
.../main/java/org/apache/helix/util/HelixUtil.java | 19 +++++
.../manager/MockParticipantManager.java | 10 ++-
3 files changed, 106 insertions(+), 22 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index a9b5bde..8c7e8f4 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -20,6 +20,8 @@ package org.apache.helix.manager.zk;
*/
import java.lang.management.ManagementFactory;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -29,6 +31,7 @@ import org.apache.helix.AccessOption;
import org.apache.helix.BaseDataAccessor;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixCloudProperty;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerProperty;
@@ -36,6 +39,8 @@ import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceInfoProvider;
import org.apache.helix.PreConnectCallback;
import org.apache.helix.PropertyKey;
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
import org.apache.helix.messaging.DefaultMessagingService;
import org.apache.helix.model.CurrentState;
import org.apache.helix.model.HelixConfigScope;
@@ -47,6 +52,7 @@ import org.apache.helix.model.StateModelDefinition;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.helix.participant.StateMachineEngine;
import org.apache.helix.participant.statemachine.ScheduledTaskStateModelFactory;
+import org.apache.helix.util.HelixUtil;
import org.apache.helix.zookeeper.api.client.HelixZkClient;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.zookeeper.datamodel.ZNRecordBucketizer;
@@ -62,6 +68,7 @@ import org.slf4j.LoggerFactory;
*/
public class ParticipantManager {
private static Logger LOG = LoggerFactory.getLogger(ParticipantManager.class);
+ private static final String CLOUD_PROCESSOR_PATH_PREFIX = "org.apache.helix.cloud.";
final HelixZkClient _zkclient;
final HelixManager _manager;
@@ -117,7 +124,6 @@ public class ParticipantManager {
* Handles a new session for a participant. The new session's id is passed in when participant
* manager is created, as it is required to prevent ephemeral node creation from session race
* condition: ephemeral node is created by an expired or unexpected session.
- *
* @throws Exception
*/
public void handleNewSession() throws Exception {
@@ -154,18 +160,31 @@ public class ParticipantManager {
}
private void joinCluster() {
- // Read cluster config and see if instance can auto join the cluster
+ // Read cluster config and see if an instance can auto join or auto register to the cluster
boolean autoJoin = false;
+ boolean autoRegistration = false;
+
+ // Read "allowParticipantAutoJoin" flag to see if an instance can auto join to the cluster
try {
- HelixConfigScope scope =
- new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(
- _manager.getClusterName()).build();
- autoJoin =
- Boolean.parseBoolean(_configAccessor.get(scope,
- ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN));
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
+ .forCluster(_manager.getClusterName()).build();
+ autoJoin = Boolean
+ .parseBoolean(_configAccessor.get(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN));
LOG.info("instance: " + _instanceName + " auto-joining " + _clusterName + " is " + autoJoin);
} catch (Exception e) {
- // autoJoin is false
+ LOG.info("auto join is false for cluster" + _clusterName);
+ }
+
+ // Read cloud config and see if an instance can auto register to the cluster
+ // Difference between auto join and auto registration is that the latter will also populate the
+ // domain information in instance config
+ try {
+ autoRegistration =
+ Boolean.valueOf(_helixManagerProperty.getHelixCloudProperty().getCloudEnabled());
+ LOG.info("instance: " + _instanceName + " auto-register " + _clusterName + " is "
+ + autoRegistration);
+ } catch (Exception e) {
+ LOG.info("auto registration is false for cluster" + _clusterName);
}
if (!ZKUtil.isInstanceSetup(_zkclient, _clusterName, _instanceName, _instanceType)) {
@@ -173,23 +192,61 @@ public class ParticipantManager {
throw new HelixException("Initial cluster structure is not set up for instance: "
+ _instanceName + ", instanceType: " + _instanceType);
} else {
- LOG.info(_instanceName + " is auto-joining cluster: " + _clusterName);
- InstanceConfig instanceConfig = new InstanceConfig(_instanceName);
- String hostName = _instanceName;
- String port = "";
- int lastPos = _instanceName.lastIndexOf("_");
- if (lastPos > 0) {
- hostName = _instanceName.substring(0, lastPos);
- port = _instanceName.substring(lastPos + 1);
+ if (!autoRegistration) {
+ LOG.info(_instanceName + " is auto-joining cluster: " + _clusterName);
+ _helixAdmin.addInstance(_clusterName, HelixUtil.composeInstanceConfig(_instanceName));
+ } else {
+ LOG.info(_instanceName + " is auto-registering cluster: " + _clusterName);
+ CloudInstanceInformation cloudInstanceInformation = getCloudInstanceInformation();
+ String domain = cloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name());
+
+ // Disable the verification for now
+ /*String cloudIdInRemote = cloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name());
+ String cloudIdInConfig = _configAccessor.getCloudConfig(_clusterName).getCloudID();
+
+ // validate that the instance is auto registering to the correct cluster
+ if (!cloudIdInRemote.equals(cloudIdInConfig)) {
+ throw new IllegalArgumentException(String.format(
+ "cloudId in config: %s is not consistent with cloudId from remote: %s. The instance is auto registering to a wrong cluster.",
+ cloudIdInConfig, cloudIdInRemote));
+ }*/
+
+ InstanceConfig instanceConfig = HelixUtil.composeInstanceConfig(_instanceName);
+ instanceConfig.setDomain(domain);
+ _helixAdmin.addInstance(_clusterName, instanceConfig);
}
- instanceConfig.setHostName(hostName);
- instanceConfig.setPort(port);
- instanceConfig.setInstanceEnabled(true);
- _helixAdmin.addInstance(_clusterName, instanceConfig);
}
}
}
+ private CloudInstanceInformation getCloudInstanceInformation() {
+ String cloudInstanceInformationProcessorName =
+ _helixManagerProperty.getHelixCloudProperty().getCloudInfoProcessorName();
+ try {
+ // fetch cloud instance information for the instance
+ String cloudInstanceInformationProcessorClassName = CLOUD_PROCESSOR_PATH_PREFIX
+ + _helixManagerProperty.getHelixCloudProperty().getCloudProvider().toLowerCase() + "."
+ + cloudInstanceInformationProcessorName;
+ Class processorClass = Class.forName(cloudInstanceInformationProcessorClassName);
+ Constructor constructor = processorClass.getConstructor(HelixCloudProperty.class);
+ CloudInstanceInformationProcessor processor = (CloudInstanceInformationProcessor) constructor
+ .newInstance(_helixManagerProperty.getHelixCloudProperty());
+ List<String> responses = processor.fetchCloudInstanceInformation();
+
+ // parse cloud instance information for the participant
+ CloudInstanceInformation cloudInstanceInformation =
+ processor.parseCloudInstanceInformation(responses);
+ return cloudInstanceInformation;
+ } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException
+ | IllegalAccessException | InvocationTargetException ex) {
+ throw new HelixException(
+ "Failed to create a new instance for the class: " + cloudInstanceInformationProcessorName,
+ ex);
+ }
+ }
+
private void createLiveInstance() {
String liveInstancePath = _keyBuilder.liveInstance(_instanceName).getPath();
LiveInstance liveInstance = new LiveInstance(_instanceName);
diff --git a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
index bfda60e..3480fb6 100644
--- a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
@@ -309,4 +309,23 @@ public final class HelixUtil {
return propertyDefaultValue;
}
+ /**
+ * Compose the config for an instance
+ * @param instanceName
+ * @return InstanceConfig
+ */
+ public static InstanceConfig composeInstanceConfig(String instanceName) {
+ InstanceConfig instanceConfig = new InstanceConfig(instanceName);
+ String hostName = instanceName;
+ String port = "";
+ int lastPos = instanceName.lastIndexOf("_");
+ if (lastPos > 0) {
+ hostName = instanceName.substring(0, lastPos);
+ port = instanceName.substring(lastPos + 1);
+ }
+ instanceConfig.setHostName(hostName);
+ instanceConfig.setPort(port);
+ instanceConfig.setInstanceEnabled(true);
+ return instanceConfig;
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java b/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
index 0036e0d..4f1b966 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/manager/MockParticipantManager.java
@@ -22,6 +22,7 @@ package org.apache.helix.integration.manager;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import org.apache.helix.HelixCloudProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.manager.zk.CallbackHandler;
import org.apache.helix.manager.zk.ZKHelixManager;
@@ -48,6 +49,7 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
protected MockMSModelFactory _msModelFactory;
protected DummyLeaderStandbyStateModelFactory _lsModelFactory;
protected DummyOnlineOfflineStateModelFactory _ofModelFactory;
+ protected HelixCloudProperty _helixCloudProperty;
public MockParticipantManager(String zkAddr, String clusterName, String instanceName) {
this(zkAddr, clusterName, instanceName, 10);
@@ -55,11 +57,17 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
public MockParticipantManager(String zkAddr, String clusterName, String instanceName,
int transDelay) {
+ this(zkAddr, clusterName, instanceName, transDelay, null);
+ }
+
+ public MockParticipantManager(String zkAddr, String clusterName, String instanceName,
+ int transDelay, HelixCloudProperty helixCloudProperty) {
super(clusterName, instanceName, InstanceType.PARTICIPANT, zkAddr);
_transDelay = transDelay;
_msModelFactory = new MockMSModelFactory(null);
_lsModelFactory = new DummyLeaderStandbyStateModelFactory(_transDelay);
_ofModelFactory = new DummyOnlineOfflineStateModelFactory(_transDelay);
+ _helixCloudProperty = helixCloudProperty;
}
public void setTransition(MockTransition transition) {
@@ -136,4 +144,4 @@ public class MockParticipantManager extends ZKHelixManager implements Runnable,
public List<CallbackHandler> getHandlers() {
return _handlers;
}
-}
+}
\ No newline at end of file
[helix] 10/12: Add construction of domain in Helix participant
logic (#876)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 7f640e44ddab6979ee533ae1896a57854ba8ad5e
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Tue Mar 10 11:10:39 2020 -0700
Add construction of domain in Helix participant logic (#876)
Add the construction of domain field in Azure cloud instance information processor. It will read the topology from AzureConstants, and construct the domain based on the format of the topology.
---
.../helix/cloud/azure/AzureCloudInstanceInformationProcessor.java | 7 ++++++-
.../main/java/org/apache/helix/manager/zk/ParticipantManager.java | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 85e2bb5..464ff55 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -142,8 +142,13 @@ public class AzureCloudInstanceInformationProcessor
String vmName = computeNode.path(INSTANCE_NAME).getTextValue();
String platformFaultDomain = computeNode.path(DOMAIN).getTextValue();
String vmssName = computeNode.path(INSTANCE_SET_NAME).getValueAsText();
+ String azureTopology = AzureConstants.AZURE_TOPOLOGY;
+ String[] parts = azureTopology.trim().split("/");
+ //The hostname will be filled in by each participant
+ String domain = parts[0] + "=" + platformFaultDomain + "," + parts[1] + "=";
+
AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
- builder.setInstanceName(vmName).setFaultDomain(platformFaultDomain)
+ builder.setInstanceName(vmName).setFaultDomain(domain)
.setInstanceSetName(vmssName);
azureCloudInstanceInformation = builder.build();
}
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
index 8c7e8f4..9eef47e 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ParticipantManager.java
@@ -199,7 +199,7 @@ public class ParticipantManager {
LOG.info(_instanceName + " is auto-registering cluster: " + _clusterName);
CloudInstanceInformation cloudInstanceInformation = getCloudInstanceInformation();
String domain = cloudInstanceInformation
- .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name());
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()) + _instanceName;
// Disable the verification for now
/*String cloudIdInRemote = cloudInstanceInformation
[helix] 01/12: Add CloudConfig code
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ba0f4e52624ffcd9cfc660fbc076842cd32b2ccf
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Nov 11 11:28:32 2019 -0800
Add CloudConfig code
In order to move toward supporting cloud environments and autoregisterations,
we need to add CloudConfig to Zookeeper.
The code regarding CloudConfig is added.
A new test has been added to check the correctness of the code.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 23 ++
.../main/java/org/apache/helix/PropertyKey.java | 11 +
.../helix/cloud/constants/CloudProvider.java | 25 ++
.../java/org/apache/helix/model/CloudConfig.java | 290 +++++++++++++++++++++
.../org/apache/helix/model/HelixConfigScope.java | 6 +-
.../model/builder/HelixConfigScopeBuilder.java | 3 +
.../apache/helix/model/cloud/TestCloudConfig.java | 204 +++++++++++++++
7 files changed, 561 insertions(+), 1 deletion(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 5d3aded..9fb9b12 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
@@ -586,6 +587,28 @@ public class ConfigAccessor {
}
/**
+ * Get CloudConfig of the given cluster.
+ * @param clusterName
+ * @return The instance of {@link CloudConfig}
+ */
+ public CloudConfig getCloudConfig(String clusterName) {
+ if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
+ throw new HelixException(
+ String.format("Failed to get config. cluster: %s is not setup.", clusterName));
+ }
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(ConfigScopeProperty.CLOUD).forCluster(clusterName).build();
+ ZNRecord record = getConfigZnRecord(scope);
+
+ if (record == null) {
+ LOG.warn("No cloud config found at {}.", scope.getZkPath());
+ return null;
+ }
+
+ return new CloudConfig(record);
+ }
+
+ /**
* Get RestConfig of the given cluster.
*
* @param clusterName The cluster
diff --git a/helix-core/src/main/java/org/apache/helix/PropertyKey.java b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
index 30305b8..73cc3f0 100644
--- a/helix-core/src/main/java/org/apache/helix/PropertyKey.java
+++ b/helix-core/src/main/java/org/apache/helix/PropertyKey.java
@@ -22,6 +22,7 @@ package org.apache.helix;
import java.util.Arrays;
import java.util.Objects;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ControllerHistory;
@@ -233,6 +234,16 @@ public class PropertyKey {
_clusterName, ConfigScopeProperty.CLUSTER.toString(), _clusterName);
}
+
+ /**
+ * Get a property key associated with this Cloud configuration
+ * @return {@link PropertyKey}
+ */
+ public PropertyKey cloudConfig() {
+ return new PropertyKey(CONFIGS, ConfigScopeProperty.CLOUD, CloudConfig.class,
+ _clusterName, ConfigScopeProperty.CLOUD.name(), _clusterName);
+ }
+
/**
* Get a property key associated with {@link InstanceConfig}
* @return {@link PropertyKey}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/constants/CloudProvider.java b/helix-core/src/main/java/org/apache/helix/cloud/constants/CloudProvider.java
new file mode 100644
index 0000000..1cb836f
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/constants/CloudProvider.java
@@ -0,0 +1,25 @@
+package org.apache.helix.cloud.constants;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public enum CloudProvider {
+ AZURE,
+ CUSTOMIZED
+}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
new file mode 100644
index 0000000..c8ab6eb
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -0,0 +1,290 @@
+package org.apache.helix.model;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixProperty;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.cloud.constants.CloudProvider;
+
+/**
+ * Cloud configurations
+ */
+public class CloudConfig extends HelixProperty {
+ /**
+ * Configurable characteristics of a cloud.
+ * NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
+ * CloudConfig.
+ */
+ public enum CloudConfigProperty {
+ CLOUD_ENABLED, // determine whether the cluster is inside cloud environment.
+ CLOUD_PROVIDER, // the environment the cluster is in, e.g. Azure, AWS, or Customized
+ CLOUD_ID, // the cloud Id that belongs to this cluster.
+
+ // If user uses Helix supported default provider, the below entries will not be shown in
+ // CloudConfig.
+ CLOUD_INFO_SOURCE, // the source for retrieving the cloud information.
+ CLOUD_INFO_PROCESSOR_NAME // the name of the function that processes the fetching and parsing of
+ // cloud information.
+ }
+
+ /* Default values */
+ private static final boolean DEFAULT_CLOUD_ENABLED = false;
+
+ /**
+ * Instantiate the CloudConfig for the cloud
+ * @param cluster
+ */
+ public CloudConfig(String cluster) {
+ super(cluster);
+ }
+
+ /**
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a cloud configuration
+ */
+ public CloudConfig(ZNRecord record) {
+ super(record);
+ }
+
+ /**
+ * Instantiate the config using each field individually.
+ * Users should use CloudConfig.Builder to create CloudConfig.
+ * @param cluster
+ * @param enabled
+ * @param cloudID
+ */
+ public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
+ List<String> cloudInfoSource, String cloudProcessorName) {
+ super(cluster);
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
+ _record
+ .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
+ }
+ }
+
+ /**
+ * Enable/Disable the CLOUD_ENABLED field.
+ * @param enabled
+ */
+ public void setCloudEnabled(boolean enabled) {
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
+ }
+
+ /**
+ * Whether CLOUD_ENABLED field is enabled or not.
+ * @return
+ */
+ public boolean isCloudEnabled() {
+ return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), false);
+ }
+
+ /**
+ * Set the cloudID field.
+ * @param cloudID
+ */
+ public void setCloudID(String cloudID) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
+ }
+
+ /**
+ * Get the CloudID field.
+ * @return CloudID
+ */
+ public String getCloudID() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name());
+ }
+
+ /**
+ * Set the CLOUD_INFO_SOURCE field.
+ * @param cloudInfoSources
+ */
+ public void setCloudInfoSource(List<String> cloudInfoSources) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources);
+ }
+
+ /**
+ * Get the CLOUD_INFO_SOURCE field.
+ * @return CLOUD_INFO_SOURCE field.
+ */
+ public List<String> getCloudInfoSources() {
+ return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
+ }
+
+ /**
+ * Set the CLOUD_INFO_PROCESSOR_NAME field.
+ * @param cloudInfoProcessorName
+ */
+ public void setCloudInfoFProcessorName(String cloudInfoProcessorName) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ cloudInfoProcessorName);
+ }
+
+ /**
+ * Get the CLOUD_INFO_PROCESSOR_NAME field.
+ * @return CLOUD_INFO_PROCESSOR_NAME field.
+ */
+ public String getCloudInfoProcessorName() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name());
+ }
+
+ /**
+ * Set the CLOUD_PROVIDER field.
+ * @param cloudProvider
+ */
+ public void setCloudProvider(CloudProvider cloudProvider) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
+ }
+
+ /**
+ * Get the CLOUD_PROVIDER field.
+ * @return CLOUD_PROVIDER field.
+ */
+ public String getCloudProvider() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
+ }
+
+ public static class Builder {
+ private String _clusterName = null;
+ private CloudProvider _cloudProvider;
+ private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED;
+ private String _cloudID;
+ private List<String> _cloudInfoSources;
+ private String _cloudInfoProcessorName;
+
+ public CloudConfig build() {
+ validate();
+ return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID,
+ _cloudInfoSources, _cloudInfoProcessorName);
+ }
+
+ /**
+ * Default constructor
+ */
+ public Builder() {
+ }
+
+ /**
+ * Constructor with Cluster Name as input
+ * @param clusterName
+ */
+ public Builder(String clusterName) {
+ _clusterName = clusterName;
+ }
+
+ /**
+ * Constructor with CloudConfig as input
+ * @param cloudConfig
+ */
+ public Builder(CloudConfig cloudConfig) {
+ _cloudEnabled = cloudConfig.isCloudEnabled();
+ _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider());
+ _cloudID = cloudConfig.getCloudID();
+ _cloudInfoSources = cloudConfig.getCloudInfoSources();
+ _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName();
+ }
+
+ public Builder setClusterName(String v) {
+ _clusterName = v;
+ return this;
+ }
+
+ public Builder setCloudEnabled(boolean isEnabled) {
+ _cloudEnabled = isEnabled;
+ return this;
+ }
+
+ public Builder setCloudProvider(CloudProvider cloudProvider) {
+ _cloudProvider = cloudProvider;
+ return this;
+ }
+
+ public Builder setCloudID(String v) {
+ _cloudID = v;
+ return this;
+ }
+
+ public Builder setCloudInfoSources(List<String> v) {
+ _cloudInfoSources = v;
+ return this;
+ }
+
+ public Builder addCloudInfoSource(String v) {
+ if (_cloudInfoSources == null) {
+ _cloudInfoSources = new ArrayList<String>();
+ }
+ _cloudInfoSources.add(v);
+ return this;
+ }
+
+ public Builder setCloudInfoProcessorName(String v) {
+ _cloudInfoProcessorName = v;
+ return this;
+ }
+
+ public String getClusterName() {
+ return _clusterName;
+ }
+
+ public CloudProvider getCloudProvider() {
+ return _cloudProvider;
+ }
+
+ public boolean getCloudEnabled() {
+ return _cloudEnabled;
+ }
+
+ public String getCloudID() {
+ return _cloudID;
+ }
+
+ public List<String> getCloudInfoSources() {
+ return _cloudInfoSources;
+ }
+
+ public String getCloudInfoProcessorName() {
+ return _cloudInfoProcessorName;
+ }
+
+ private void validate() {
+ if (_cloudEnabled) {
+ if (_cloudID == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The CloudID is missing from the config.");
+ }
+ if (_cloudProvider == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (_cloudProvider == CloudProvider.CUSTOMIZED) {
+ if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
index 7c5c91d..8d814c5 100644
--- a/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
+++ b/helix-core/src/main/java/org/apache/helix/model/HelixConfigScope.java
@@ -36,7 +36,8 @@ public class HelixConfigScope {
RESOURCE(2, 0),
PARTITION(2, 1),
CONSTRAINT(2, 0),
- REST(2, 0);
+ REST(2, 0),
+ CLOUD(2, 0);
final int _zkPathArgNum;
final int _mapKeyArgNum;
@@ -78,12 +79,15 @@ public class HelixConfigScope {
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}");
template.addEntry(ConfigScopeProperty.PARTITION, 2,
"/{clusterName}/CONFIGS/RESOURCE/{resourceName}");
+ template.addEntry(ConfigScopeProperty.CLOUD, 2,
+ "/{clusterName}/CONFIGS/CLOUD/{clusterName}");
// get children
template.addEntry(ConfigScopeProperty.CLUSTER, 1, "/{clusterName}/CONFIGS/CLUSTER");
template.addEntry(ConfigScopeProperty.PARTICIPANT, 1, "/{clusterName}/CONFIGS/PARTICIPANT");
template.addEntry(ConfigScopeProperty.RESOURCE, 1, "/{clusterName}/CONFIGS/RESOURCE");
template.addEntry(ConfigScopeProperty.REST, 2, "/{clusterName}/CONFIGS/REST/{clusterName}");
+ template.addEntry(ConfigScopeProperty.CLOUD, 1, "/{clusterName}/CONFIGS/CLOUD");
}
final ConfigScopeProperty _type;
diff --git a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
index b1a7cf6..78ed074 100644
--- a/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
+++ b/helix-core/src/main/java/org/apache/helix/model/builder/HelixConfigScopeBuilder.java
@@ -126,6 +126,9 @@ public class HelixConfigScopeBuilder {
case REST:
scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
break;
+ case CLOUD:
+ scope = new HelixConfigScope(_type, Arrays.asList(_clusterName, _clusterName), null);
+ break;
default:
break;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
new file mode 100644
index 0000000..ee83011
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
@@ -0,0 +1,204 @@
+package org.apache.helix.model.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import org.apache.helix.ConfigAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.PropertyKey.Builder;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.manager.zk.ZKHelixDataAccessor;
+import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CloudConfig;
+import org.apache.helix.cloud.constants.CloudProvider;
+import java.util.List;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestCloudConfig extends ZkUnitTestBase {
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testCloudConfigNonExistentCluster() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ // Read CloudConfig from Zookeeper and get exception since cluster in not setup yet
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ }
+
+ @Test(dependsOnMethods = "testCloudConfigNonExistentCluster")
+ public void testCloudConfigNull() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ // Read CloudConfig from Zookeeper
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ // since CloudConfig is not written to ZooKeeper, the output should be null
+ Assert.assertNull(cloudConfigFromZk);
+ }
+
+ @Test(dependsOnMethods = "testCloudConfigNull")
+ public void testCloudConfig() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+
+ // Create dummy CloudConfig object
+ CloudConfig cloudConfig = new CloudConfig(clusterName);
+ cloudConfig.setCloudEnabled(true);
+ cloudConfig.setCloudProvider(CloudProvider.AZURE);
+ cloudConfig.setCloudID("TestID");
+ List<String> infoURL = new ArrayList<String>();
+ infoURL.add("TestURL");
+ cloudConfig.setCloudInfoSource(infoURL);
+ cloudConfig.setCloudInfoFProcessorName("TestProcessor");
+
+ // Write the CloudConfig to Zookeeper
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1);
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testUnverifiedCloudConfigBuilder() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ // Verify will fail because cloudID has net been defined.
+ CloudConfig cloudConfig = builder.build();
+ }
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testUnverifiedCloudConfigBuilderEmptySources() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudID("TestID");
+ List<String> emptyList = new ArrayList<String>();
+ builder.setCloudInfoSources(emptyList);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+ }
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testUnverifiedCloudConfigBuilderWithoutProcessor() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudID("TestID");
+ List<String> testList = new ArrayList<String>();
+ builder.setCloudInfoSources(testList);
+ builder.addCloudInfoSource("TestURL");
+ CloudConfig cloudConfig = builder.build();
+ }
+
+ @Test(dependsOnMethods = "testCloudConfig")
+ public void testCloudConfigBuilder() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL0");
+ builder.addCloudInfoSource("TestURL1");
+ builder.setCloudInfoProcessorName("TestProcessor");
+
+ // Check builder getter methods
+ Assert.assertTrue(builder.getCloudEnabled());
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED);
+ Assert.assertEquals(builder.getClusterName(), clusterName);
+ Assert.assertEquals(builder.getCloudID(), "TestID");
+ List<String> listUrlFromBuilder = builder.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromBuilder.size(), 2);
+ Assert.assertEquals(listUrlFromBuilder.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromBuilder.get(1), "TestURL1");
+ Assert.assertEquals(builder.getCloudInfoProcessorName(), "TestProcessor");
+
+ CloudConfig cloudConfig = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromZk.get(1), "TestURL1");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+ @Test(dependsOnMethods = "testCloudConfigBuilder")
+ public void testCloudConfigBuilderAzureProvider() {
+ String className = getShortClassName();
+ String clusterName = "CLUSTER_" + className;
+ TestHelper.setupEmptyCluster(_gZkClient, clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ builder.setCloudEnabled(true);
+ builder.setCloudProvider(CloudProvider.AZURE);
+ builder.setCloudID("TestID");
+ builder.setCloudInfoProcessorName("TestProcessor");
+
+ // Check builder getter methods
+ Assert.assertTrue(builder.getCloudEnabled());
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE);
+
+ CloudConfig cloudConfig = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null.
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ }
+}
[helix] 02/12: add Helix cloud interface and implementation
skeleton methods
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 64ca2388133b88ceb5da81d49418ba3c16ad7e96
Author: Meng Zhang <mn...@mnzhang-ld2.linkedin.biz>
AuthorDate: Wed Nov 13 09:43:04 2019 -0800
add Helix cloud interface and implementation skeleton methods
---
.../helix/api/cloud/CloudInstanceInformation.java | 40 +++++++++++
.../cloud/CloudInstanceInformationProcessor.java | 41 ++++++++++++
.../cloud/azure/AzureCloudInstanceInformation.java | 77 ++++++++++++++++++++++
.../AzureCloudInstanceInformationProcessor.java | 56 ++++++++++++++++
4 files changed, 214 insertions(+)
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformation.java
new file mode 100644
index 0000000..b2429f9
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformation.java
@@ -0,0 +1,40 @@
+package org.apache.helix.api.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Generic interface for cloud instance information
+ */
+public interface CloudInstanceInformation {
+ /**
+ * Get the the value of a specific cloud instance field by name
+ * @return the value of the field
+ */
+ String get(String key);
+
+ /**
+ * The enum contains all the required cloud instance field in Helix
+ */
+ enum CloudInstanceField {
+ INSTANCE_NAME,
+ FAULT_DOMAIN,
+ INSTANCE_SET_NAME
+ }
+}
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..a365777
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
@@ -0,0 +1,41 @@
+package org.apache.helix.api.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+
+/**
+ * Generic interface to fetch and parse cloud instance information
+ */
+public interface CloudInstanceInformationProcessor<T extends Object> {
+
+ /**
+ * Get the raw cloud instance information
+ * @return raw cloud instance information
+ */
+ List<T> fetchCloudInstanceInformation();
+
+ /**
+ * Parse the raw cloud instance information in responses and compose required cloud instance information
+ * @return required cloud instance information
+ */
+ CloudInstanceInformation parseCloudInstanceInformation(List<T> responses);
+}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
new file mode 100644
index 0000000..511f3e3
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -0,0 +1,77 @@
+package org.apache.helix.cloud.azure;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Map;
+
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+
+
+public class AzureCloudInstanceInformation implements CloudInstanceInformation {
+ private Map<String, String> _cloudInstanceInfoMap;
+
+ /**
+ * Instantiate the AzureCloudInstanceInformation using each field individually.
+ * Users should use AzureCloudInstanceInformation.Builder to create information.
+ * @param cloudInstanceInfoMap
+ */
+ protected AzureCloudInstanceInformation(Map<String, String> cloudInstanceInfoMap) {
+ _cloudInstanceInfoMap = cloudInstanceInfoMap;
+ }
+
+ @Override
+ public String get(String key) {
+ return _cloudInstanceInfoMap.get(key);
+ }
+
+ public static class Builder {
+ private Map<String, String> _cloudInstanceInfoMap = null;
+
+ public AzureCloudInstanceInformation build() {
+ return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
+ }
+
+ /**
+ * Default constructor
+ */
+ public Builder() {
+ }
+
+ public Builder setInstanceName(String v) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_NAME.name(), v);
+ return this;
+ }
+
+ public Builder setFaultDomain(String v) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.FAULT_DOMAIN.name(), v);
+ return this;
+ }
+
+ public Builder setInstanceSetName(String v) {
+ _cloudInstanceInfoMap.put(CloudInstanceField.INSTANCE_SET_NAME.name(), v);
+ return this;
+ }
+
+ public Builder setCloudInstanceInfoField(String key, String value) {
+ _cloudInstanceInfoMap.put(key, value);
+ return this;
+ }
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..84a102c
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -0,0 +1,56 @@
+package org.apache.helix.cloud.azure;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
+
+
+public class AzureCloudInstanceInformationProcessor implements CloudInstanceInformationProcessor<String> {
+
+ public AzureCloudInstanceInformationProcessor() {
+ }
+
+ /**
+ * fetch the raw Azure cloud instance information
+ * @return raw Azure cloud instance information
+ */
+ @Override
+ public List<String> fetchCloudInstanceInformation() {
+ List<String> response = new ArrayList<>();
+ //TODO: implement the fetching logic
+ return response;
+ }
+
+ /**
+ * Parse raw Azure cloud instance information.
+ * @return required azure cloud instance information
+ */
+ @Override
+ public AzureCloudInstanceInformation parseCloudInstanceInformation(List<String> responses) {
+ AzureCloudInstanceInformation azureCloudInstanceInformation = null;
+ //TODO: implement the parsing logic
+ return azureCloudInstanceInformation;
+ }
+}
+
+
[helix] 03/12: Add java API to create cluster with CloudConfig
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 9f6cc4b19a5446ebf6b910b7a9872e0b4fb6f3e3
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Tue Nov 26 15:26:10 2019 -0800
Add java API to create cluster with CloudConfig
In this commit the below APIs have been added.
1- AddCluster with CloudCOnfig API.
2- addCloudConfig to existing cluster.
3- Update CloudConfig to update existing Cloud Config.
Several tests have been added to test these APIs.
---
.../main/java/org/apache/helix/ConfigAccessor.java | 4 +-
.../src/main/java/org/apache/helix/HelixAdmin.java | 15 ++-
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 28 ++++
.../java/org/apache/helix/model/CloudConfig.java | 149 +++++++--------------
.../java/org/apache/helix/tools/ClusterSetup.java | 17 ++-
.../java/org/apache/helix/TestConfigAccessor.java | 53 +++++++-
.../apache/helix/manager/zk/TestZkHelixAdmin.java | 70 ++++++++++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 11 ++
.../apache/helix/model/cloud/TestCloudConfig.java | 60 ++++++---
.../org/apache/helix/tools/TestClusterSetup.java | 88 +++++++++++-
10 files changed, 363 insertions(+), 132 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
index 9fb9b12..e09ae5b 100644
--- a/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/ConfigAccessor.java
@@ -592,7 +592,7 @@ public class ConfigAccessor {
* @return The instance of {@link CloudConfig}
*/
public CloudConfig getCloudConfig(String clusterName) {
- if (!ZKUtil.isClusterSetup(clusterName, zkClient)) {
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
throw new HelixException(
String.format("Failed to get config. cluster: %s is not setup.", clusterName));
}
@@ -605,7 +605,7 @@ public class ConfigAccessor {
return null;
}
- return new CloudConfig(record);
+ return new CloudConfig.Builder(record).build();
}
/**
diff --git a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
index 423f879..511a763 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixAdmin.java
@@ -22,7 +22,7 @@ package org.apache.helix;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
import org.apache.helix.model.ConstraintItem;
@@ -382,6 +382,19 @@ public interface HelixAdmin {
void dropResource(String clusterName, String resourceName);
/**
+ * Add cloud config to the cluster.
+ * @param clusterName
+ * @param cloudConfig
+ */
+ void addCloudConfig(String clusterName, CloudConfig cloudConfig);
+
+ /**
+ * Remove the Cloud Config for specific cluster
+ * @param clusterName
+ */
+ void removeCloudConfig(String clusterName);
+
+ /**
* Get a list of state model definitions in a cluster
* @param clusterName
* @return
diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
index 77d8103..923fe26 100644
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZKHelixAdmin.java
@@ -55,6 +55,7 @@ import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.controller.rebalancer.util.WagedValidationUtil;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -1030,6 +1031,33 @@ public class ZKHelixAdmin implements HelixAdmin {
}
@Override
+ public void addCloudConfig(String clusterName, CloudConfig cloudConfig) {
+ logger.info("Add CloudConfig to cluster {}, CloudConfig is {}.", clusterName,
+ cloudConfig.toString());
+
+ if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
+ throw new HelixException("cluster " + clusterName + " is not setup yet");
+ }
+
+ CloudConfig.Builder builder = new CloudConfig.Builder(cloudConfig);
+ CloudConfig cloudConfigBuilder = builder.build();
+
+ ZKHelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfigBuilder);
+ }
+
+ @Override
+ public void removeCloudConfig(String clusterName) {
+ logger.info("Remove Cloud Config for cluster {}.", clusterName);
+ HelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
+ Builder keyBuilder = accessor.keyBuilder();
+ accessor.removeProperty(keyBuilder.cloudConfig());
+ }
+
+ @Override
public List<String> getStateModelDefs(String clusterName) {
return _zkClient.getChildren(PropertyPathBuilder.stateModelDef(clusterName));
}
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index c8ab6eb..f6c279c 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -30,6 +30,9 @@ import org.apache.helix.cloud.constants.CloudProvider;
* Cloud configurations
*/
public class CloudConfig extends HelixProperty {
+
+ public static final String CLOUD_CONFIG_KW = "CloudConfig";
+
/**
* Configurable characteristics of a cloud.
* NOTE: Do NOT use this field name directly, use its corresponding getter/setter in the
@@ -52,39 +55,22 @@ public class CloudConfig extends HelixProperty {
/**
* Instantiate the CloudConfig for the cloud
- * @param cluster
*/
- public CloudConfig(String cluster) {
- super(cluster);
+ private CloudConfig() {
+ super(CLOUD_CONFIG_KW);
}
/**
- * Instantiate with a pre-populated record
- * @param record a ZNRecord corresponding to a cloud configuration
+ * The constructor from the ZNRecord.
+ * @param record
*/
- public CloudConfig(ZNRecord record) {
- super(record);
+ private CloudConfig(ZNRecord record) {
+ super(CLOUD_CONFIG_KW);
+ _record.setSimpleFields(record.getSimpleFields());
+ _record.setListFields(record.getListFields());
+ _record.setMapFields(record.getMapFields());
}
- /**
- * Instantiate the config using each field individually.
- * Users should use CloudConfig.Builder to create CloudConfig.
- * @param cluster
- * @param enabled
- * @param cloudID
- */
- public CloudConfig(String cluster, boolean enabled, CloudProvider cloudProvider, String cloudID,
- List<String> cloudInfoSource, String cloudProcessorName) {
- super(cluster);
- _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), enabled);
- _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
- _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
- if (cloudProvider.equals(CloudProvider.CUSTOMIZED)) {
- _record
- .setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(), cloudProcessorName);
- _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSource);
- }
- }
/**
* Enable/Disable the CLOUD_ENABLED field.
@@ -135,15 +121,6 @@ public class CloudConfig extends HelixProperty {
}
/**
- * Set the CLOUD_INFO_PROCESSOR_NAME field.
- * @param cloudInfoProcessorName
- */
- public void setCloudInfoFProcessorName(String cloudInfoProcessorName) {
- _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
- cloudInfoProcessorName);
- }
-
- /**
* Get the CLOUD_INFO_PROCESSOR_NAME field.
* @return CLOUD_INFO_PROCESSOR_NAME field.
*/
@@ -152,14 +129,6 @@ public class CloudConfig extends HelixProperty {
}
/**
- * Set the CLOUD_PROVIDER field.
- * @param cloudProvider
- */
- public void setCloudProvider(CloudProvider cloudProvider) {
- _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
- }
-
- /**
* Get the CLOUD_PROVIDER field.
* @return CLOUD_PROVIDER field.
*/
@@ -167,32 +136,28 @@ public class CloudConfig extends HelixProperty {
return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
}
+
public static class Builder {
- private String _clusterName = null;
- private CloudProvider _cloudProvider;
- private boolean _cloudEnabled = DEFAULT_CLOUD_ENABLED;
- private String _cloudID;
- private List<String> _cloudInfoSources;
- private String _cloudInfoProcessorName;
+ private ZNRecord _record;
public CloudConfig build() {
validate();
- return new CloudConfig(_clusterName, _cloudEnabled, _cloudProvider, _cloudID,
- _cloudInfoSources, _cloudInfoProcessorName);
+ return new CloudConfig(_record);
}
/**
* Default constructor
*/
public Builder() {
+ _record = new ZNRecord(CLOUD_CONFIG_KW);
}
/**
- * Constructor with Cluster Name as input
- * @param clusterName
+ * Instantiate with a pre-populated record
+ * @param record a ZNRecord corresponding to a cloud configuration
*/
- public Builder(String clusterName) {
- _clusterName = clusterName;
+ public Builder(ZNRecord record) {
+ _record = record;
}
/**
@@ -200,89 +165,75 @@ public class CloudConfig extends HelixProperty {
* @param cloudConfig
*/
public Builder(CloudConfig cloudConfig) {
- _cloudEnabled = cloudConfig.isCloudEnabled();
- _cloudProvider = CloudProvider.valueOf(cloudConfig.getCloudProvider());
- _cloudID = cloudConfig.getCloudID();
- _cloudInfoSources = cloudConfig.getCloudInfoSources();
- _cloudInfoProcessorName = cloudConfig.getCloudInfoProcessorName();
- }
-
- public Builder setClusterName(String v) {
- _clusterName = v;
- return this;
+ _record = cloudConfig.getRecord();
}
public Builder setCloudEnabled(boolean isEnabled) {
- _cloudEnabled = isEnabled;
+ _record.setBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(), isEnabled);
return this;
}
public Builder setCloudProvider(CloudProvider cloudProvider) {
- _cloudProvider = cloudProvider;
+ _record.setSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name(), cloudProvider.name());
return this;
}
- public Builder setCloudID(String v) {
- _cloudID = v;
+ public Builder setCloudID(String cloudID) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_ID.name(), cloudID);
return this;
}
- public Builder setCloudInfoSources(List<String> v) {
- _cloudInfoSources = v;
+ public Builder setCloudInfoSources(List<String> cloudInfoSources) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSources);
return this;
}
- public Builder addCloudInfoSource(String v) {
- if (_cloudInfoSources == null) {
- _cloudInfoSources = new ArrayList<String>();
+ public Builder addCloudInfoSource(String cloudInfoSource) {
+ if (_record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name()) == null) {
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), new ArrayList<String>());
}
- _cloudInfoSources.add(v);
+ List<String> cloudInfoSourcesList = _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
+ cloudInfoSourcesList.add(cloudInfoSource);
+ _record.setListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name(), cloudInfoSourcesList);
return this;
}
- public Builder setCloudInfoProcessorName(String v) {
- _cloudInfoProcessorName = v;
+ public Builder setCloudInfoProcessorName(String cloudInfoProcessorName) {
+ _record.setSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ cloudInfoProcessorName);
return this;
}
- public String getClusterName() {
- return _clusterName;
- }
-
- public CloudProvider getCloudProvider() {
- return _cloudProvider;
+ public String getCloudProvider() {
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_PROVIDER.name());
}
public boolean getCloudEnabled() {
- return _cloudEnabled;
+ return _record.getBooleanField(CloudConfigProperty.CLOUD_ENABLED.name(),
+ DEFAULT_CLOUD_ENABLED);
}
public String getCloudID() {
- return _cloudID;
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_ID.name());
}
public List<String> getCloudInfoSources() {
- return _cloudInfoSources;
+ return _record.getListField(CloudConfigProperty.CLOUD_INFO_SOURCE.name());
}
public String getCloudInfoProcessorName() {
- return _cloudInfoProcessorName;
+ return _record.getSimpleField(CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name());
}
private void validate() {
- if (_cloudEnabled) {
- if (_cloudID == null) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. The CloudID is missing from the config.");
- }
- if (_cloudProvider == null) {
+ if (this.getCloudProvider() == null) {
+ throw new HelixException(
+ "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
+ } else if (this.getCloudProvider().equals(CloudProvider.CUSTOMIZED.name())) {
+ if (this.getCloudInfoProcessorName() == null || this.getCloudInfoSources() == null
+ || this.getCloudInfoSources().size() == 0) {
throw new HelixException(
- "This Cloud Configuration is Invalid. The Cloud Provider is missing from the config.");
- } else if (_cloudProvider == CloudProvider.CUSTOMIZED) {
- if (_cloudInfoProcessorName == null || _cloudInfoSources == null || _cloudInfoSources.size() == 0) {
- throw new HelixException(
- "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
- }
+ "This Cloud Configuration is Invalid. CUSTOMIZED provider has been chosen without defining CloudInfoProcessorName or CloudInfoSources");
}
}
}
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 7ac20a1..1d51a74 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -41,6 +41,8 @@ import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.SystemPropertyKeys;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
@@ -48,6 +50,7 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.zookeeper.api.client.HelixZkClient;
import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
import org.apache.helix.model.BuiltInStateModelDefinitions;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -176,15 +179,23 @@ public class ClusterSetup {
}
}
- public void addCluster(String clusterName, boolean overwritePrevious) {
+ public void addCluster(String clusterName, boolean overwritePrevious, CloudConfig cloudConfig)
+ throws HelixException {
_admin.addCluster(clusterName, overwritePrevious);
-
for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) {
addStateModelDef(clusterName, def.getStateModelDefinition().getId(),
- def.getStateModelDefinition(), overwritePrevious);
+ def.getStateModelDefinition(), overwritePrevious);
+ }
+
+ if (cloudConfig != null) {
+ _admin.addCloudConfig(clusterName, cloudConfig);
}
}
+ public void addCluster(String clusterName, boolean overwritePrevious) {
+ addCluster(clusterName, overwritePrevious, null);
+ }
+
public void activateCluster(String clusterName, String grandCluster, boolean enable) {
if (enable) {
_admin.addClusterToGrandCluster(clusterName, grandCluster);
diff --git a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
index 345cba3..4452c23 100644
--- a/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/TestConfigAccessor.java
@@ -19,10 +19,13 @@ package org.apache.helix;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.HelixConfigScope;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
@@ -30,6 +33,7 @@ import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.RESTConfig;
import org.apache.helix.model.builder.ConfigScopeBuilder;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.helix.tools.ClusterSetup;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -72,9 +76,8 @@ public class TestConfigAccessor extends ZkUnitTestBase {
Assert.assertEquals(resourceConfigValue, "resourceConfigValue");
// partition scope config
- ConfigScope partitionScope =
- new ConfigScopeBuilder().forCluster(clusterName).forResource("testResource")
- .forPartition("testPartition").build();
+ ConfigScope partitionScope = new ConfigScopeBuilder().forCluster(clusterName)
+ .forResource("testResource").forPartition("testPartition").build();
configAccessor.set(partitionScope, "partitionConfigKey", "partitionConfigValue");
String partitionConfigValue = configAccessor.get(partitionScope, "partitionConfigKey");
Assert.assertEquals(partitionConfigValue, "partitionConfigValue");
@@ -276,4 +279,48 @@ public class TestConfigAccessor extends ZkUnitTestBase {
"Fail to delete REST config. cluster: " + anotherClusterName + " is NOT setup.");
}
}
+
+ public void testUpdateCloudConfig() throws Exception {
+ ClusterSetup _clusterSetup = new ClusterSetup(ZK_ADDR);
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestCloudID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+
+ // Change the processor name and check if processor name has been changed in Zookeeper.
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessor2");
+ cloudConfigInit = cloudConfigInitBuilder.build();
+ ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCloudConfig(clusterName, cloudConfigInit);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor2");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
index ac1cc09..234f2c3 100644
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkHelixAdmin.java
@@ -30,6 +30,7 @@ import java.util.Map;
import com.google.common.collect.ImmutableMap;
import org.apache.helix.BaseDataAccessor;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
@@ -46,6 +47,9 @@ import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
import org.apache.helix.examples.MasterSlaveStateModelFactory;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.model.ClusterConfig;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.examples.MasterSlaveStateModelFactory;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ClusterConstraints.ConstraintAttribute;
import org.apache.helix.model.ClusterConstraints.ConstraintType;
@@ -614,4 +618,70 @@ public class TestZkHelixAdmin extends ZkUnitTestBase {
IdealState is = admin.getResourceIdealState(clusterName, testResourcePrefix);
Assert.assertEquals(is.getRebalancerClassName(), WagedRebalancer.class.getName());
}
+
+ @Test
+ public void testAddCloudConfig() {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+
+ CloudConfig.Builder builder = new CloudConfig.Builder();
+ builder.setCloudEnabled(true);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL");
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+
+ admin.addCloudConfig(clusterName, cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+ }
+
+
+ @Test
+ public void testRemoveCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
+ admin.addCluster(clusterName, true);
+
+ CloudConfig.Builder builder = new CloudConfig.Builder();
+ builder.setCloudEnabled(true);
+ builder.setCloudID("TestID");
+ builder.addCloudInfoSource("TestURL");
+ builder.setCloudProvider(CloudProvider.CUSTOMIZED);
+ builder.setCloudInfoProcessorName("TestProcessor");
+ CloudConfig cloudConfig = builder.build();
+
+ admin.addCloudConfig(clusterName, cloudConfig);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
+
+ // Remove Cloud Config and make sure it has been removed from Zookeeper
+ admin.removeCloudConfig(clusterName);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNull(cloudConfigFromZk);
+ }
}
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
index b3502eb..69fdb7b 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockHelixAdmin.java
@@ -31,6 +31,7 @@ import org.apache.helix.HelixManager;
import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.PropertyType;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ClusterConstraints;
import org.apache.helix.model.ConstraintItem;
@@ -314,6 +315,16 @@ public class MockHelixAdmin implements HelixAdmin {
}
+ @Override
+ public void addCloudConfig(String clusterName, CloudConfig cloudConfig) {
+
+ }
+
+ @Override
+ public void removeCloudConfig(String clusterName) {
+
+ }
+
@Override public List<String> getStateModelDefs(String clusterName) {
return null;
}
diff --git a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
index ee83011..01945de 100644
--- a/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
+++ b/helix-core/src/test/java/org/apache/helix/model/cloud/TestCloudConfig.java
@@ -57,20 +57,22 @@ public class TestCloudConfig extends ZkUnitTestBase {
}
@Test(dependsOnMethods = "testCloudConfigNull")
- public void testCloudConfig() {
+ public void testCloudConfig() throws Exception {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
// Create dummy CloudConfig object
- CloudConfig cloudConfig = new CloudConfig(clusterName);
- cloudConfig.setCloudEnabled(true);
- cloudConfig.setCloudProvider(CloudProvider.AZURE);
- cloudConfig.setCloudID("TestID");
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
List<String> infoURL = new ArrayList<String>();
infoURL.add("TestURL");
- cloudConfig.setCloudInfoSource(infoURL);
- cloudConfig.setCloudInfoFProcessorName("TestProcessor");
+ cloudConfigBuilder.setCloudInfoSources(infoURL);
+ cloudConfigBuilder.setCloudInfoProcessorName("TestProcessor");
+
+ CloudConfig cloudConfig = cloudConfigBuilder.build();
// Write the CloudConfig to Zookeeper
ZKHelixDataAccessor accessor =
@@ -84,8 +86,8 @@ public class TestCloudConfig extends ZkUnitTestBase {
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+
Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().size(), 1);
- Assert.assertEquals(cloudConfigFromZk.getCloudInfoSources().get(0), "TestURL");
Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessor");
}
@@ -93,7 +95,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
public void testUnverifiedCloudConfigBuilder() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
// Verify will fail because cloudID has net been defined.
CloudConfig cloudConfig = builder.build();
@@ -103,7 +105,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
public void testUnverifiedCloudConfigBuilderEmptySources() {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -115,9 +117,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
@Test(expectedExceptions = HelixException.class)
public void testUnverifiedCloudConfigBuilderWithoutProcessor() {
- String className = getShortClassName();
- String clusterName = "CLUSTER_" + className;
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -132,7 +132,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.CUSTOMIZED);
builder.setCloudID("TestID");
@@ -142,8 +142,7 @@ public class TestCloudConfig extends ZkUnitTestBase {
// Check builder getter methods
Assert.assertTrue(builder.getCloudEnabled());
- Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED);
- Assert.assertEquals(builder.getClusterName(), clusterName);
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
Assert.assertEquals(builder.getCloudID(), "TestID");
List<String> listUrlFromBuilder = builder.getCloudInfoSources();
Assert.assertEquals(listUrlFromBuilder.size(), 2);
@@ -175,15 +174,14 @@ public class TestCloudConfig extends ZkUnitTestBase {
String className = getShortClassName();
String clusterName = "CLUSTER_" + className;
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
- CloudConfig.Builder builder = new CloudConfig.Builder(clusterName);
+ CloudConfig.Builder builder = new CloudConfig.Builder();
builder.setCloudEnabled(true);
builder.setCloudProvider(CloudProvider.AZURE);
builder.setCloudID("TestID");
- builder.setCloudInfoProcessorName("TestProcessor");
// Check builder getter methods
Assert.assertTrue(builder.getCloudEnabled());
- Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE);
+ Assert.assertEquals(builder.getCloudProvider(), CloudProvider.AZURE.name());
CloudConfig cloudConfig = builder.build();
@@ -198,7 +196,29 @@ public class TestCloudConfig extends ZkUnitTestBase {
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
- // Since CloudProvider is not CUSTOMIZED, CloudInfoProcessor will be null.
+ // Since user does not set the CloudInfoProcessorName, this field will be null.
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+
+ // Checking the set method in CloudConfig
+ cloudConfig.setCloudEnabled(false);
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
+
+ cloudConfig.setCloudEnabled(true);
+ cloudConfig.setCloudID("TestID2");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL0");
+ sourceList.add("TestURL1");
+ cloudConfig.setCloudInfoSource(sourceList);
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID2");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL0");
+ Assert.assertEquals(listUrlFromZk.get(1), "TestURL1");
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index 2a3e04a..bd5449f 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -19,10 +19,13 @@ package org.apache.helix.tools;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.List;
import org.apache.helix.BaseDataAccessor;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey;
@@ -31,15 +34,15 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.AssertJUnit;
import org.testng.annotations.AfterClass;
@@ -48,8 +51,6 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
public class TestClusterSetup extends ZkUnitTestBase {
- private static Logger LOG = LoggerFactory.getLogger(TestClusterSetup.class);
-
protected static final String CLUSTER_NAME = "TestClusterSetup";
protected static final String TEST_DB = "TestDB";
protected static final String INSTANCE_PREFIX = "instance_";
@@ -436,4 +437,83 @@ public class TestClusterSetup extends ZkUnitTestBase {
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
+
+ @Test(expectedExceptions = HelixException.class)
+ public void testAddClusterWithInvalidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+
+ // Since setCloudInfoProcessorName is missing, this add cluster call will throw an exception
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig")
+ public void testAddClusterWithValidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestID");
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ cloudConfigInitBuilder.setCloudInfoSources(sourceList);
+ cloudConfigInitBuilder.setCloudInfoProcessorName("TestProcessorName");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.CUSTOMIZED);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
+
+
+ @Test(dependsOnMethods = "testAddClusterWithValidCloudConfig")
+ public void testAddClusterAzureProvider() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ CloudConfig.Builder cloudConfigInitBuilder = new CloudConfig.Builder();
+ cloudConfigInitBuilder.setCloudEnabled(true);
+ cloudConfigInitBuilder.setCloudID("TestID");
+ cloudConfigInitBuilder.setCloudProvider(CloudProvider.AZURE);
+
+ CloudConfig cloudConfigInit = cloudConfigInitBuilder.build();
+
+ _clusterSetup.addCluster(clusterName, false, cloudConfigInit);
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+
+ // Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
+ Assert.assertNull(listUrlFromZk);
+ Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
}
[helix] 11/12: Change the REST call for delete CloudConfig (#882)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit ab3c4da31b36820c0739bfa71f6694f852767693
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 13 13:18:07 2020 -0700
Change the REST call for delete CloudConfig (#882)
Change the delete CloudConfig
Use DELETE instead of POST for deletion of the CloudConfig in REST.
---
.../server/resources/helix/ClusterAccessor.java | 12 ++++++---
.../helix/rest/server/TestClusterAccessor.java | 31 +++++++++++++++-------
2 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 27cebda..3966d7c 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -715,6 +715,14 @@ public class ClusterAccessor extends AbstractHelixResource {
return notFound();
}
+ @DELETE
+ @Path("{clusterId}/cloudconfig")
+ public Response deleteCloudConfig(@PathParam("clusterId") String clusterId) {
+ HelixAdmin admin = getHelixAdmin();
+ admin.removeCloudConfig(clusterId);
+ return OK();
+ }
+
@POST
@Path("{clusterId}/cloudconfig")
public Response updateCloudConfig(@PathParam("clusterId") String clusterId,
@@ -748,10 +756,6 @@ public class ClusterAccessor extends AbstractHelixResource {
}
try {
switch (command) {
- case delete: {
- admin.removeCloudConfig(clusterId);
- }
- break;
case update: {
try {
CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 6eae926..78a7a2c 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -868,19 +868,32 @@ public class TestClusterAccessor extends AbstractTestClass {
@Test(dependsOnMethods = "testAddCloudConfig")
public void testDeleteCloudConfig() throws IOException {
System.out.println("Start test :" + TestHelper.getTestMethodName());
- _gSetupTool.addCluster("TestCloud", true);
- String urlBase = "clusters/TestCloud/cloudconfig/";
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
- Map<String, String> map1 = new HashMap<>();
- map1.put("command", AbstractResource.Command.delete.name());
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+ // Read CloudConfig from Zookeeper and make sure it has been created
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertNotNull(cloudConfigFromZk);
+ String urlBase = "clusters/" + clusterName + "/cloudconfig/";
- ZNRecord record = new ZNRecord("TestCloud");
- post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
- Response.Status.OK.getStatusCode());
+ delete(urlBase, Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
- CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ _configAccessor = new ConfigAccessor(ZK_ADDR);
+ cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertNull(cloudConfigFromZk);
System.out.println("End test :" + TestHelper.getTestMethodName());
[helix] 08/12: add one more test for auto registration (#806)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit a32f38784a69d6304a7eb12e10c29cfdad9981ff
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Wed Feb 26 10:05:41 2020 -0800
add one more test for auto registration (#806)
Add a test case for participant auto registers to cluster in cloud environment. Note that the final step is expected to fail as the test is not running in a cloud environment.
---
.../paticipant/TestInstanceAutoJoin.java | 62 +++++++++++++++++++---
1 file changed, 54 insertions(+), 8 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
index f0ffc63..5d5ec46 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceAutoJoin.java
@@ -1,16 +1,22 @@
package org.apache.helix.integration.paticipant;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.integration.common.ZkStandAloneCMTestBase;
import org.apache.helix.integration.manager.MockParticipantManager;
import org.apache.helix.manager.zk.ZKHelixManager;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ConfigScope;
import org.apache.helix.model.IdealState.RebalanceMode;
import org.apache.helix.model.builder.ConfigScopeBuilder;
import org.testng.Assert;
import org.testng.annotations.Test;
+import static org.testng.Assert.*;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -32,14 +38,15 @@ import org.testng.annotations.Test;
public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
String db2 = TEST_DB + "2";
+ String db3 = TEST_DB + "3";
@Test
public void testInstanceAutoJoin() throws Exception {
HelixManager manager = _participants[0];
HelixDataAccessor accessor = manager.getHelixDataAccessor();
- _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "OnlineOffline", RebalanceMode.FULL_AUTO
- + "");
+ _gSetupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "OnlineOffline",
+ RebalanceMode.FULL_AUTO + "");
_gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 1);
String instance2 = "localhost_279699";
@@ -50,8 +57,8 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
Thread.sleep(500);
// Assert.assertFalse(result._thread.isAlive());
- Assert.assertTrue(null == manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2)));
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2)));
ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
@@ -63,15 +70,54 @@ public class TestInstanceAutoJoin extends ZkStandAloneCMTestBase {
Thread.sleep(500);
// Assert.assertTrue(result._thread.isAlive() || result2._thread.isAlive());
for (int i = 0; i < 20; i++) {
- if (null == manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2))) {
+ if (null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2))) {
Thread.sleep(100);
} else
break;
}
- Assert.assertTrue(null != manager.getHelixDataAccessor().getProperty(
- accessor.keyBuilder().liveInstance(instance2)));
+ Assert.assertTrue(null != manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance2)));
newParticipant.syncStop();
}
+
+ @Test(dependsOnMethods = "testInstanceAutoJoin")
+ public void testAutoRegistrationShouldFailWhenWaitingResponse() throws Exception {
+ // Create CloudConfig object and add to config
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
+ CloudConfig cloudConfig = cloudConfigBuilder.build();
+
+ HelixManager manager = _participants[0];
+ HelixDataAccessor accessor = manager.getHelixDataAccessor();
+
+ _gSetupTool.addResourceToCluster(CLUSTER_NAME, db3, 60, "OnlineOffline",
+ RebalanceMode.FULL_AUTO + "");
+ _gSetupTool.rebalanceStorageCluster(CLUSTER_NAME, db3, 1);
+ String instance3 = "localhost_279700";
+
+ ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
+
+ manager.getConfigAccessor().set(scope, ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, "true");
+ // Write the CloudConfig to Zookeeper
+ PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ accessor.setProperty(keyBuilder.cloudConfig(), cloudConfig);
+
+ MockParticipantManager autoParticipant =
+ new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance3);
+ autoParticipant.syncStart();
+
+ Assert.assertTrue(null == manager.getHelixDataAccessor()
+ .getProperty(accessor.keyBuilder().liveInstance(instance3)));
+ try {
+ manager.getConfigAccessor().getInstanceConfig(CLUSTER_NAME, instance3);
+ fail(
+ "Exception should be thrown because the instance cannot be added to the cluster due to the disconnection with Azure endpoint");
+ } catch (HelixException e) {
+
+ }
+ }
}
[helix] 09/12: Change the cluster creation logic (#872)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 912e7943e3c2a6e70f2427bc3295e52828f37b73
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Fri Mar 6 13:15:26 2020 -0800
Change the cluster creation logic (#872)
* Change the cluster creation logic
In this commit, minor modifications have been added which:
1- Update topology information in the Cluster Config
if cloud is enabled for Azure environment.
2- Tests have been change accordingly.
3- The deprecated APIs have been changed in the test to
follow new APIs.
---
.../org/apache/helix/cloud/azure/AzureConstants.java | 6 ++++++
.../main/java/org/apache/helix/tools/ClusterSetup.java | 13 ++++++++++++-
.../java/org/apache/helix/tools/TestClusterSetup.java | 14 +++++++++++---
.../apache/helix/rest/server/TestClusterAccessor.java | 16 +++++++++++-----
4 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
new file mode 100644
index 0000000..b93f345
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureConstants.java
@@ -0,0 +1,6 @@
+package org.apache.helix.cloud.azure;
+
+public class AzureConstants {
+ public static final String AZURE_TOPOLOGY = "/faultDomain/hostname";
+ public static final String AZURE_FAULT_ZONE_TYPE = "faultDomain";
+}
\ No newline at end of file
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
index 1d51a74..881a8f6 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterSetup.java
@@ -36,12 +36,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.PropertyKey.Builder;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
-import org.apache.helix.SystemPropertyKeys;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -189,6 +190,16 @@ public class ClusterSetup {
if (cloudConfig != null) {
_admin.addCloudConfig(clusterName, cloudConfig);
+ // If cloud is enabled and Cloud Provider is Azure, populated the Topology information in cluster config
+ if (cloudConfig.isCloudEnabled()
+ && cloudConfig.getCloudProvider().equals(CloudProvider.AZURE.name())) {
+ ConfigAccessor configAccessor = new ConfigAccessor(_zkServerAddress);
+ ClusterConfig clusterConfig = new ClusterConfig(clusterName);
+ clusterConfig.setTopology(AzureConstants.AZURE_TOPOLOGY);
+ clusterConfig.setTopologyAwareEnabled(true);
+ clusterConfig.setFaultZoneType(AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ configAccessor.updateClusterConfig(clusterName, clusterConfig);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
index bd5449f..ee5ae07 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterSetup.java
@@ -34,12 +34,14 @@ import org.apache.helix.PropertyPathBuilder;
import org.apache.helix.TestHelper;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.model.CloudConfig;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
@@ -355,7 +357,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
// add fake liveInstance
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR));
Builder keyBuilder = new Builder(clusterName);
LiveInstance liveInstance = new LiveInstance("localhost_12918");
liveInstance.setSessionId("session_0");
@@ -421,7 +423,7 @@ public class TestClusterSetup extends ZkUnitTestBase {
ClusterSetup.processCommandLineArgs(new String[] {
"--zkSvr", ZK_ADDR, "--enableResource", clusterName, "TestDB0", "false"
});
- BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
+ BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(ZK_ADDR);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
@@ -505,12 +507,18 @@ public class TestClusterSetup extends ZkUnitTestBase {
_clusterSetup.addCluster(clusterName, false, cloudConfigInit);
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestID");
List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ // Since it is Azure, topology information should have been populated.
+ ClusterConfig clusterConfig = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfig.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfig.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfig.isTopologyAwareEnabled());
+
// Since provider is not customized, CloudInfoSources and CloudInfoProcessorName will be null.
Assert.assertNull(listUrlFromZk);
Assert.assertNull(cloudConfigFromZk.getCloudInfoProcessorName());
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index 09406e1..6eae926 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -39,6 +39,7 @@ import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
import org.apache.helix.model.RESTConfig;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.cloud.azure.AzureConstants;
import org.apache.helix.controller.rebalancer.DelayedAutoRebalancer;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.waged.WagedRebalancer;
@@ -665,11 +666,16 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+
+ ClusterConfig clusterConfigFromZk = _configAccessor.getClusterConfig(clusterName);
+ Assert.assertEquals(clusterConfigFromZk.getTopology(), AzureConstants.AZURE_TOPOLOGY);
+ Assert.assertEquals(clusterConfigFromZk.getFaultZoneType(), AzureConstants.AZURE_FAULT_ZONE_TYPE);
+ Assert.assertTrue(clusterConfigFromZk.isTopologyAwareEnabled());
}
@Test(dependsOnMethods = "testAddClusterWithCloudConfig")
@@ -740,7 +746,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -770,7 +776,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.CREATED.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -829,7 +835,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and check the content
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
@@ -873,7 +879,7 @@ public class TestClusterAccessor extends AbstractTestClass {
Response.Status.OK.getStatusCode());
// Read CloudConfig from Zookeeper and make sure it has been removed
- ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ ConfigAccessor _configAccessor = new ConfigAccessor(ZK_ADDR);
CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
Assert.assertNull(cloudConfigFromZk);
[helix] 04/12: Add REST API for Cluster Creation with CloudConfig
(#675)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 3bbeb3198c17ec06dfb6542d2918ac9745f4891c
Author: Ali Reza Zamani Zadeh Najari <an...@linkedin.com>
AuthorDate: Mon Jan 20 01:16:45 2020 -0800
Add REST API for Cluster Creation with CloudConfig (#675)
This commit contains the relevant REST APIs for cluster creation.
This change allows user to create cluster with CloudConfig.
Multiple tests have been added in order to check the functionality of REST calls.
---
.../java/org/apache/helix/model/CloudConfig.java | 4 +-
.../server/resources/helix/ClusterAccessor.java | 139 +++++++++-
.../helix/rest/server/TestClusterAccessor.java | 297 +++++++++++++++++++++
3 files changed, 437 insertions(+), 3 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
index f6c279c..c1ffa61 100644
--- a/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
+++ b/helix-core/src/main/java/org/apache/helix/model/CloudConfig.java
@@ -21,10 +21,12 @@ package org.apache.helix.model;
import java.util.ArrayList;
import java.util.List;
+
import org.apache.helix.HelixException;
import org.apache.helix.HelixProperty;
-import org.apache.helix.ZNRecord;
import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+
/**
* Cloud configurations
diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
index 762210a..27cebda 100644
--- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
+++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/ClusterAccessor.java
@@ -48,6 +48,7 @@ import org.apache.helix.model.RESTConfig;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
import org.apache.helix.manager.zk.ZKUtil;
import org.apache.helix.zookeeper.api.client.HelixZkClient;
+import org.apache.helix.model.CloudConfig;
import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.ControllerHistory;
import org.apache.helix.model.HelixConfigScope;
@@ -134,17 +135,38 @@ public class ClusterAccessor extends AbstractHelixResource {
@PUT
@Path("{clusterId}")
public Response createCluster(@PathParam("clusterId") String clusterId,
- @DefaultValue("false") @QueryParam("recreate") String recreate) {
+ @DefaultValue("false") @QueryParam("recreate") String recreate,
+ @DefaultValue("false") @QueryParam("addCloudConfig") String addCloudConfig,
+ String content) {
+
boolean recreateIfExists = Boolean.valueOf(recreate);
+ boolean cloudConfigIncluded = Boolean.valueOf(addCloudConfig);
+
ClusterSetup clusterSetup = getClusterSetup();
+ CloudConfig cloudConfig = null;
+ if (cloudConfigIncluded) {
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ LOG.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+ try {
+ cloudConfig = new CloudConfig.Builder(record).build();
+ clusterSetup.addCluster(clusterId, recreateIfExists, cloudConfig);
+ } catch (Exception ex) {
+ LOG.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ }
+ }
try {
clusterSetup.addCluster(clusterId, recreateIfExists);
} catch (Exception ex) {
LOG.error("Failed to create cluster {}. Exception: {}.", clusterId, ex);
return serverError(ex);
}
-
return created();
}
@@ -642,6 +664,119 @@ public class ClusterAccessor extends AbstractHelixResource {
return ZKUtil.isClusterSetup(cluster, zkClient);
}
+ @PUT
+ @Path("{clusterId}/cloudconfig")
+ public Response addCloudConfig(@PathParam("clusterId") String clusterId, String content) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound("Cluster is not properly setup!");
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ LOG.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+
+ try {
+ CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
+ admin.addCloudConfig(clusterId, cloudConfig);
+ } catch (HelixException ex) {
+ LOG.error("Error in adding a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ } catch (Exception ex) {
+ LOG.error("Cannot add CloudConfig to cluster: " + clusterId, ex);
+ return serverError(ex);
+ }
+
+ return OK();
+ }
+
+ @GET
+ @Path("{clusterId}/cloudconfig")
+ public Response getCloudConfig(@PathParam("clusterId") String clusterId) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound();
+ }
+
+ ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
+ CloudConfig cloudConfig = configAccessor.getCloudConfig(clusterId);
+
+ if (cloudConfig != null) {
+ return JSONRepresentation(cloudConfig.getRecord());
+ }
+
+ return notFound();
+ }
+
+ @POST
+ @Path("{clusterId}/cloudconfig")
+ public Response updateCloudConfig(@PathParam("clusterId") String clusterId,
+ @QueryParam("command") String commandStr, String content) {
+
+ HelixZkClient zkClient = getHelixZkClient();
+ if (!ZKUtil.isClusterSetup(clusterId, zkClient)) {
+ return notFound();
+ }
+
+ // Here to update cloud config
+ Command command;
+ if (commandStr == null || commandStr.isEmpty()) {
+ command = Command.update; // Default behavior
+ } else {
+ try {
+ command = getCommand(commandStr);
+ } catch (HelixException ex) {
+ return badRequest(ex.getMessage());
+ }
+ }
+
+ HelixAdmin admin = getHelixAdmin();
+
+ ZNRecord record;
+ try {
+ record = toZNRecord(content);
+ } catch (IOException e) {
+ LOG.error("Failed to deserialize user's input " + content + ", Exception: " + e);
+ return badRequest("Input is not a vaild ZNRecord!");
+ }
+ try {
+ switch (command) {
+ case delete: {
+ admin.removeCloudConfig(clusterId);
+ }
+ break;
+ case update: {
+ try {
+ CloudConfig cloudConfig = new CloudConfig.Builder(record).build();
+ admin.removeCloudConfig(clusterId);
+ admin.addCloudConfig(clusterId, cloudConfig);
+ } catch (HelixException ex) {
+ LOG.error("Error in updating a CloudConfig to cluster: " + clusterId, ex);
+ return badRequest(ex.getMessage());
+ } catch (Exception ex) {
+ LOG.error("Cannot update CloudConfig for cluster: " + clusterId, ex);
+ return serverError(ex);
+ }
+ }
+ break;
+ default:
+ return badRequest("Unsupported command " + commandStr);
+ }
+ } catch (Exception ex) {
+ LOG.error("Failed to " + command + " cloud config, cluster " + clusterId + " new config: "
+ + content + ", Exception: " + ex);
+ return serverError(ex);
+ }
+ return OK();
+ }
+
/**
* Reads HISTORY ZNode from the metadata store and generates a Map object that contains the
* pertinent history entries depending on the history type.
diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
index b073d22..09406e1 100644
--- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
+++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestClusterAccessor.java
@@ -33,6 +33,7 @@ import javax.ws.rs.core.Response;
import com.google.common.collect.ImmutableMap;
import com.sun.research.ws.wadl.HTTPMethods;
+import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
@@ -50,6 +51,8 @@ import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.MaintenanceSignal;
+import org.apache.helix.model.CloudConfig;
+import org.apache.helix.cloud.constants.CloudProvider;
import org.apache.helix.rest.common.HelixRestNamespace;
import org.apache.helix.rest.server.auditlog.AuditLog;
import org.apache.helix.rest.server.resources.AbstractResource;
@@ -642,6 +645,300 @@ public class TestClusterAccessor extends AbstractTestClass {
System.out.println("End test :" + TestHelper.getTestMethodName());
}
+ @Test(dependsOnMethods = "testActivateSuperCluster")
+ public void testAddClusterWithCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithCloudConfig")
+ public void testAddClusterWithInvalidCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider has not been defined. Result of this rest call will be BAD_REQUEST.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCloudConfig")
+ public void testAddClusterWithInvalidCustomizedCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider is customized. CLOUD_INFO_PROCESSOR_NAME and CLOUD_INFO_SOURCE fields are
+ // required.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithInvalidCustomizedCloudConfig")
+ public void testAddClusterWithValidCustomizedCloudConfig() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ List<String> sourceList = new ArrayList<String>();
+ sourceList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), sourceList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ // Cloud Provider is customized. CLOUD_INFO_PROCESSOR_NAME and CLOUD_INFO_SOURCE fields are
+ // required.
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ }
+
+ @Test(dependsOnMethods = "testAddClusterWithValidCustomizedCloudConfig")
+ public void testAddClusterWithCloudConfigDisabledCloud() throws Exception {
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ String clusterName = className + "_" + methodName;
+
+ ZNRecord record = new ZNRecord("testZnode");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), false);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ Map<String, String> map = new HashMap<>();
+ map.put("addCloudConfig", "true");
+
+ put("clusters/" + clusterName, map,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.CREATED.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig(clusterName);
+ Assert.assertFalse(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.AZURE.name());
+ }
+
+
+ @Test(dependsOnMethods = "testAddClusterWithCloudConfigDisabledCloud")
+ public void testAddCloudConfigNonExistedCluster() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessor");
+
+ // Not found since the cluster is not setup yet.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.NOT_FOUND.getStatusCode());
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCloudConfigNonExistedCluster")
+ public void testAddCloudConfig() throws Exception {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+
+ // Bad request since Processor has not been defined.
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.BAD_REQUEST.getStatusCode());
+
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ // Now response should be OK since all fields are set
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and check the content
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ Assert.assertTrue(cloudConfigFromZk.isCloudEnabled());
+ Assert.assertEquals(cloudConfigFromZk.getCloudID(), "TestCloudID");
+ List<String> listUrlFromZk = cloudConfigFromZk.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromZk.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigFromZk.getCloudInfoProcessorName(), "TestProcessorName");
+ Assert.assertEquals(cloudConfigFromZk.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+
+ // Now test the getCloudConfig method.
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CloudConfig cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ CloudConfig cloudConfigZk = _configAccessor.getCloudConfig("TestCloud");
+
+ // Check that the CloudConfig from Zk and REST get method are equal
+ Assert.assertEquals(cloudConfigRest, cloudConfigZk);
+
+ // Check the fields individually
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromRest = cloudConfigRest.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromRest.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigRest.getCloudInfoProcessorName(), "TestProcessorName");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testAddCloudConfig")
+ public void testDeleteCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", AbstractResource.Command.delete.name());
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ post(urlBase, map1, Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Read CloudConfig from Zookeeper and make sure it has been removed
+ ConfigAccessor _configAccessor = new ConfigAccessor(_gZkClient);
+ CloudConfig cloudConfigFromZk = _configAccessor.getCloudConfig("TestCloud");
+ Assert.assertNull(cloudConfigFromZk);
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
+ @Test(dependsOnMethods = "testDeleteCloudConfig")
+ public void testUpdateCloudConfig() throws IOException {
+ System.out.println("Start test :" + TestHelper.getTestMethodName());
+ _gSetupTool.addCluster("TestCloud", true);
+ String urlBase = "clusters/TestCloud/cloudconfig/";
+
+ ZNRecord record = new ZNRecord("TestCloud");
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudID");
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.AZURE.name());
+
+ // Fist add CloudConfig to the cluster
+ put(urlBase, null,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Now get the Cloud Config and make sure the information is correct
+ String body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ ZNRecord recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ CloudConfig cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudID");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.AZURE.name());
+
+ // Now put new information in the ZNRecord
+ record.setBooleanField(CloudConfig.CloudConfigProperty.CLOUD_ENABLED.name(), true);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_PROVIDER.name(),
+ CloudProvider.CUSTOMIZED.name());
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_ID.name(), "TestCloudIdNew");
+ List<String> testList = new ArrayList<String>();
+ testList.add("TestURL");
+ record.setListField(CloudConfig.CloudConfigProperty.CLOUD_INFO_SOURCE.name(), testList);
+ record.setSimpleField(CloudConfig.CloudConfigProperty.CLOUD_INFO_PROCESSOR_NAME.name(),
+ "TestProcessorName");
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("command", AbstractResource.Command.update.name());
+
+ post(urlBase, map1,
+ Entity.entity(OBJECT_MAPPER.writeValueAsString(record), MediaType.APPLICATION_JSON_TYPE),
+ Response.Status.OK.getStatusCode());
+
+ // Now get the Cloud Config and make sure the information has been updated
+ body = get(urlBase, null, Response.Status.OK.getStatusCode(), true);
+
+ recordFromRest = new ObjectMapper().reader(ZNRecord.class).readValue(body);
+ cloudConfigRest = new CloudConfig.Builder(recordFromRest).build();
+ Assert.assertTrue(cloudConfigRest.isCloudEnabled());
+ Assert.assertEquals(cloudConfigRest.getCloudID(), "TestCloudIdNew");
+ Assert.assertEquals(cloudConfigRest.getCloudProvider(), CloudProvider.CUSTOMIZED.name());
+ List<String> listUrlFromRest = cloudConfigRest.getCloudInfoSources();
+ Assert.assertEquals(listUrlFromRest.get(0), "TestURL");
+ Assert.assertEquals(cloudConfigRest.getCloudInfoProcessorName(), "TestProcessorName");
+
+ System.out.println("End test :" + TestHelper.getTestMethodName());
+ }
+
private ClusterConfig getClusterConfigFromRest(String cluster) throws IOException {
String body = get("clusters/" + cluster + "/configs", null, Response.Status.OK.getStatusCode(), true);
[helix] 06/12: Implement Azure cloud instance information processor
(#698)
Posted by jx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jxue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 079483b37eb59b1b10a91b92f30468de7da73547
Author: zhangmeng916 <56...@users.noreply.github.com>
AuthorDate: Fri Feb 7 15:24:29 2020 -0800
Implement Azure cloud instance information processor (#698)
Implement Azure cloud instance information processor. The processor performs the functions of fetching and parsing instance information from Azure cloud environment. The endpoint that the processor queries is a globally standard url, called Azure Instance Metadata Service.
---
helix-core/pom.xml | 5 +
.../cloud/CloudInstanceInformationProcessor.java | 1 -
.../cloud/azure/AzureCloudInstanceInformation.java | 11 +-
.../AzureCloudInstanceInformationProcessor.java | 113 +++++++++++++++++++--
.../org/apache/helix/cloud/MockHttpClient.java | 53 ++++++++++
...TestAzureCloudInstanceInformationProcessor.java | 69 +++++++++++++
helix-core/src/test/resources/AzureResponse.json | 104 +++++++++++++++++++
helix-rest/pom.xml | 5 -
8 files changed, 343 insertions(+), 18 deletions(-)
diff --git a/helix-core/pom.xml b/helix-core/pom.xml
index 18113e1..6aaa816 100644
--- a/helix-core/pom.xml
+++ b/helix-core/pom.xml
@@ -163,6 +163,11 @@ under the License.
<artifactId>metrics-core</artifactId>
<version>3.2.3</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.8</version>
+ </dependency>
</dependencies>
<build>
<resources>
diff --git a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
index a365777..9ff6b00 100644
--- a/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/api/cloud/CloudInstanceInformationProcessor.java
@@ -21,7 +21,6 @@ package org.apache.helix.api.cloud;
import java.util.List;
-
/**
* Generic interface to fetch and parse cloud instance information
*/
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
index f7fd657..1fef205 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformation.java
@@ -19,17 +19,17 @@ package org.apache.helix.cloud.azure;
* under the License.
*/
+import java.util.HashMap;
import java.util.Map;
import org.apache.helix.api.cloud.CloudInstanceInformation;
-
public class AzureCloudInstanceInformation implements CloudInstanceInformation {
private Map<String, String> _cloudInstanceInfoMap;
/**
* Instantiate the AzureCloudInstanceInformation using each field individually.
- * Users should use AzureCloudInstanceInformation.Builder to create information.
+ * Users should use AzureCloudInstanceInformation.Builder to set field information.
* @param cloudInstanceInfoMap
*/
protected AzureCloudInstanceInformation(Map<String, String> cloudInstanceInfoMap) {
@@ -42,10 +42,11 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
}
public static class Builder {
- private Map<String, String> _cloudInstanceInfoMap = null;
+
+ private final Map<String, String> _cloudInstanceInfoMap = new HashMap<>();
public AzureCloudInstanceInformation build() {
- return new AzureCloudInstanceInformation(_cloudInstanceInfoMap);
+ return new AzureCloudInstanceInformation(new HashMap<>(_cloudInstanceInfoMap));
}
public Builder setInstanceName(String name) {
@@ -68,4 +69,4 @@ public class AzureCloudInstanceInformation implements CloudInstanceInformation {
return this;
}
}
-}
\ No newline at end of file
+}
diff --git a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
index 84a102c..85e2bb5 100644
--- a/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
+++ b/helix-core/src/main/java/org/apache/helix/cloud/azure/AzureCloudInstanceInformationProcessor.java
@@ -19,38 +19,137 @@ package org.apache.helix.cloud.azure;
* under the License.
*/
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
+import javax.net.ssl.SSLException;
+import org.apache.helix.HelixCloudProperty;
+import org.apache.helix.HelixException;
import org.apache.helix.api.cloud.CloudInstanceInformationProcessor;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.EntityUtils;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+public class AzureCloudInstanceInformationProcessor
+ implements CloudInstanceInformationProcessor<String> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AzureCloudInstanceInformationProcessor.class);
+ private final CloseableHttpClient _closeableHttpClient;
+ private final HelixCloudProperty _helixCloudProperty;
+ private final String COMPUTE = "compute";
+ private final String INSTANCE_NAME = "vmId";
+ private final String DOMAIN = "platformFaultDomain";
+ private final String INSTANCE_SET_NAME = "vmScaleSetName";
-public class AzureCloudInstanceInformationProcessor implements CloudInstanceInformationProcessor<String> {
+ public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty) {
+ _helixCloudProperty = helixCloudProperty;
- public AzureCloudInstanceInformationProcessor() {
+ RequestConfig requestConifg = RequestConfig.custom()
+ .setConnectionRequestTimeout((int) helixCloudProperty.getCloudRequestTimeout())
+ .setConnectTimeout((int) helixCloudProperty.getCloudConnectionTimeout()).build();
+
+ HttpRequestRetryHandler httpRequestRetryHandler =
+ (IOException exception, int executionCount, HttpContext context) -> {
+ LOG.warn("Execution count: " + executionCount + ".", exception);
+ return !(executionCount >= helixCloudProperty.getCloudMaxRetry()
+ || exception instanceof InterruptedIOException
+ || exception instanceof UnknownHostException || exception instanceof SSLException);
+ };
+
+ //TODO: we should regularize the way how httpClient should be used throughout Helix. e.g. Helix-rest could also use in the same way
+ _closeableHttpClient = HttpClients.custom().setDefaultRequestConfig(requestConifg)
+ .setRetryHandler(httpRequestRetryHandler).build();
}
/**
- * fetch the raw Azure cloud instance information
+ * This constructor is for unit test purpose only.
+ * User could provide helixCloudProperty and a mocked http client to test the functionality of
+ * this class.
+ */
+ public AzureCloudInstanceInformationProcessor(HelixCloudProperty helixCloudProperty,
+ CloseableHttpClient closeableHttpClient) {
+ _helixCloudProperty = helixCloudProperty;
+ _closeableHttpClient = closeableHttpClient;
+ }
+
+ /**
+ * Fetch raw Azure cloud instance information based on the urls provided
* @return raw Azure cloud instance information
*/
@Override
public List<String> fetchCloudInstanceInformation() {
List<String> response = new ArrayList<>();
- //TODO: implement the fetching logic
+ for (String url : _helixCloudProperty.getCloudInfoSources()) {
+ response.add(getAzureCloudInformationFromUrl(url));
+ }
return response;
}
/**
+ * Query Azure Instance Metadata Service to get the instance(VM) information
+ * @return raw Azure cloud instance information
+ */
+ private String getAzureCloudInformationFromUrl(String url) {
+ HttpGet httpGet = new HttpGet(url);
+ httpGet.setHeader("Metadata", "true");
+
+ try {
+ CloseableHttpResponse response = _closeableHttpClient.execute(httpGet);
+ if (response == null || response.getStatusLine().getStatusCode() != 200) {
+ String errorMsg = String.format(
+ "Failed to get an HTTP Response for the request. Response: {}. Status code: {}",
+ (response == null ? "NULL" : response.getStatusLine().getReasonPhrase()),
+ response.getStatusLine().getStatusCode());
+ throw new HelixException(errorMsg);
+ }
+ String responseString = EntityUtils.toString(response.getEntity());
+ LOG.info("VM instance information query result: {}", responseString);
+ return responseString;
+ } catch (IOException e) {
+ throw new HelixException(
+ String.format("Failed to get Azure cloud instance information from url {}", url), e);
+ }
+ }
+
+ /**
* Parse raw Azure cloud instance information.
* @return required azure cloud instance information
*/
@Override
public AzureCloudInstanceInformation parseCloudInstanceInformation(List<String> responses) {
AzureCloudInstanceInformation azureCloudInstanceInformation = null;
- //TODO: implement the parsing logic
+ if (responses.size() > 1) {
+ throw new HelixException("Multiple responses are not supported for Azure now");
+ }
+ String response = responses.get(0);
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ JsonNode jsonNode = mapper.readTree(response);
+ JsonNode computeNode = jsonNode.path(COMPUTE);
+ if (!computeNode.isMissingNode()) {
+ String vmName = computeNode.path(INSTANCE_NAME).getTextValue();
+ String platformFaultDomain = computeNode.path(DOMAIN).getTextValue();
+ String vmssName = computeNode.path(INSTANCE_SET_NAME).getValueAsText();
+ AzureCloudInstanceInformation.Builder builder = new AzureCloudInstanceInformation.Builder();
+ builder.setInstanceName(vmName).setFaultDomain(platformFaultDomain)
+ .setInstanceSetName(vmssName);
+ azureCloudInstanceInformation = builder.build();
+ }
+ } catch (IOException e) {
+ throw new HelixException(String.format("Error in parsing cloud instance information: {}", response, e));
+ }
return azureCloudInstanceInformation;
}
}
-
-
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java b/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
new file mode 100644
index 0000000..03e687e
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/cloud/MockHttpClient.java
@@ -0,0 +1,53 @@
+package org.apache.helix.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.InputStream;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+
+/**
+ * Mock a http client and provide response using resource file. This is for unit test purpose only.
+ */
+public class MockHttpClient {
+ protected CloseableHttpClient createMockHttpClient(String file) throws Exception {
+ InputStream responseInputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(file);
+ HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+ StatusLine statusLine = Mockito.mock(StatusLine.class);
+
+ CloseableHttpResponse mockCloseableHttpResponse = Mockito.mock(CloseableHttpResponse.class);
+ CloseableHttpClient mockCloseableHttpClient = Mockito.mock(CloseableHttpClient.class);
+
+ Mockito.when(httpEntity.getContent()).thenReturn(responseInputStream);
+ Mockito.when(mockCloseableHttpClient.execute(Matchers.any(HttpGet.class))).thenReturn(mockCloseableHttpResponse);
+ Mockito.when(mockCloseableHttpResponse.getEntity()).thenReturn(httpEntity);
+ Mockito.when(mockCloseableHttpResponse.getStatusLine()).thenReturn(statusLine);
+ Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+
+ return mockCloseableHttpClient;
+ }
+}
\ No newline at end of file
diff --git a/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
new file mode 100644
index 0000000..10ba42d
--- /dev/null
+++ b/helix-core/src/test/java/org/apache/helix/cloud/TestAzureCloudInstanceInformationProcessor.java
@@ -0,0 +1,69 @@
+package org.apache.helix.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+import org.apache.helix.HelixCloudProperty;
+import org.apache.helix.api.cloud.CloudInstanceInformation;
+import org.apache.helix.cloud.azure.AzureCloudInstanceInformation;
+import org.apache.helix.cloud.azure.AzureCloudInstanceInformationProcessor;
+import org.apache.helix.cloud.constants.CloudProvider;
+import org.apache.helix.model.CloudConfig;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+/**
+ * Unit test for {@link AzureCloudInstanceInformationProcessor}
+ */
+public class TestAzureCloudInstanceInformationProcessor extends MockHttpClient {
+
+ @Test()
+ public void testAzureCloudInstanceInformationProcessing() throws Exception {
+ String responseFile = "AzureResponse.json";
+
+ CloudConfig.Builder cloudConfigBuilder = new CloudConfig.Builder();
+ cloudConfigBuilder.setCloudEnabled(true);
+ cloudConfigBuilder.setCloudProvider(CloudProvider.AZURE);
+ cloudConfigBuilder.setCloudID("TestID");
+ HelixCloudProperty helixCloudProperty = new HelixCloudProperty(cloudConfigBuilder.build());
+ AzureCloudInstanceInformationProcessor processor = new AzureCloudInstanceInformationProcessor(
+ helixCloudProperty, createMockHttpClient(responseFile));
+ List<String> response = processor.fetchCloudInstanceInformation();
+
+ Assert.assertEquals(response.size(), 1);
+ Assert.assertNotNull(response.get(0));
+
+ // Verify the response from mock http client
+ AzureCloudInstanceInformation azureCloudInstanceInformation =
+ processor.parseCloudInstanceInformation(response);
+ Assert.assertEquals(azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.FAULT_DOMAIN.name()), "2");
+ Assert.assertEquals(
+ azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_SET_NAME.name()),
+ "test-helix");
+ Assert.assertEquals(
+ azureCloudInstanceInformation
+ .get(CloudInstanceInformation.CloudInstanceField.INSTANCE_NAME.name()),
+ "d2b921cc-c16c-41f7-a86d-a445eac6ec26");
+ }
+}
diff --git a/helix-core/src/test/resources/AzureResponse.json b/helix-core/src/test/resources/AzureResponse.json
new file mode 100644
index 0000000..dfe13ad
--- /dev/null
+++ b/helix-core/src/test/resources/AzureResponse.json
@@ -0,0 +1,104 @@
+{
+ "compute": {
+ "azEnvironment": "AzurePublicCloud",
+ "customData": "",
+ "location": "southcentralus",
+ "name": "test-helix_1",
+ "offer": "",
+ "osType": "Linux",
+ "placementGroupId": "81e605b2-a807-48ee-a84a-63c76a9c9543",
+ "plan": {
+ "name": "",
+ "product": "",
+ "publisher": ""
+ },
+ "platformFaultDomain": "2",
+ "platformUpdateDomain": "2",
+ "provider": "Microsoft.Compute",
+ "publicKeys": [],
+ "publisher": "",
+ "resourceGroupName": "scus-lpsazureei1-app-rg",
+ "resourceId": "/subscriptions/c9a251d8-1272-4c0f-8055-8271bbc1d677/resourceGroups/scus-lpsazureei1-app-rg/providers/Microsoft.Compute/virtualMachines/test-helix_2",
+ "sku": "",
+ "storageProfile": {
+ "dataDisks": [],
+ "imageReference": {
+ "id": "/subscriptions/7dd5a659-67c4-441c-ac0b-d48b7a029668/resourceGroups/scus-infra-app-rg/providers/Microsoft.Compute/galleries/pieimagerepo/images/FastCOP4/versions/190924.1.1",
+ "offer": "",
+ "publisher": "",
+ "sku": "",
+ "version": ""
+ },
+ "osDisk": {
+ "caching": "ReadWrite",
+ "createOption": "FromImage",
+ "diskSizeGB": "32",
+ "encryptionSettings": {
+ "enabled": "false"
+ },
+ "image": {
+ "uri": ""
+ },
+ "managedDisk": {
+ "id": "/subscriptions/c9a251d8-1272-4c0f-8055-8271bbc1d677/resourceGroups/scus-lpsazureei1-app-rg/providers/Microsoft.Compute/disks/test-helix_test-helix_2_OsDisk_1_124c3534b8e848e296ec22b24d44c027",
+ "storageAccountType": "Standard_LRS"
+ },
+ "name": "test-helix_test-helix_2_OsDisk_1_124c3534b8e848e296ec22b24d44c027",
+ "osType": "Linux",
+ "vhd": {
+ "uri": ""
+ },
+ "writeAcceleratorEnabled": "false"
+ }
+ },
+ "subscriptionId": "c9a251d8-1272-4c0f-8055-8271bbc1d677",
+ "tags": "automation:terraform;environment:dev;module:lid-vmss;moduleVersion:0.0.1",
+ "tagsList": [
+ {
+ "name": "automation",
+ "value": "terraform"
+ },
+ {
+ "name": "environment",
+ "value": "dev"
+ },
+ {
+ "name": "module",
+ "value": "lid-vmss"
+ },
+ {
+ "name": "moduleVersion",
+ "value": "0.0.1"
+ }
+ ],
+ "version": "",
+ "vmId": "d2b921cc-c16c-41f7-a86d-a445eac6ec26",
+ "vmScaleSetName": "test-helix",
+ "vmSize": "Standard_D16s_v3",
+ "zone": ""
+ },
+ "network": {
+ "interface": [
+ {
+ "ipv4": {
+ "ipAddress": [
+ {
+ "privateIpAddress": "10.3.64.12",
+ "publicIpAddress": ""
+ }
+ ],
+ "subnet": [
+ {
+ "address": "10.3.64.0",
+ "prefix": "19"
+ }
+ ]
+ },
+ "ipv6": {
+ "ipAddress": []
+ },
+ "macAddress": "000D3A769010"
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/helix-rest/pom.xml b/helix-rest/pom.xml
index e3421d5..d5b3ee5 100644
--- a/helix-rest/pom.xml
+++ b/helix-rest/pom.xml
@@ -69,11 +69,6 @@ under the License.
<version>3.8.1</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.8</version>
- </dependency>
- <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>9.1.0.RC0</version>