You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/04/12 19:43:26 UTC
[ambari] 01/02: Revert "[AMBARI-23493] Decommission Datanode
doesn't work (dsen)"
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 5ab00009037ff820d312a747547226225d6b5911
Author: Aravindan Vijayan <av...@hortonworks.com>
AuthorDate: Thu Apr 12 12:07:06 2018 -0700
Revert "[AMBARI-23493] Decommission Datanode doesn't work (dsen)"
This reverts commit 675ca9a64f70fae517aeefcadcbab137a239922a.
---
.../AmbariCustomCommandExecutionHelper.java | 16 -------------
.../org/apache/ambari/server/utils/StageUtils.java | 28 ++++++++++++++++++++++
.../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 5 +---
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 5 +---
.../controller/AmbariManagementControllerTest.java | 9 -------
.../apache/ambari/server/utils/StageUtilsTest.java | 4 ++++
6 files changed, 34 insertions(+), 33 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index d396d12..61b14c4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -129,7 +129,6 @@ public class AmbariCustomCommandExecutionHelper {
public final static String DECOM_INCLUDED_HOSTS = "included_hosts";
public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts";
- public final static String ALL_DECOMMISSIONED_HOSTS = "all_decommissioned_hosts";
public final static String DECOM_SLAVE_COMPONENT = "slave_type";
public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only";
public final static String UPDATE_FILES_ONLY = "update_files_only";
@@ -1057,10 +1056,6 @@ public class AmbariCustomCommandExecutionHelper {
}
Map<String, String> commandParams = new HashMap<>();
-
- commandParams.put(ALL_DECOMMISSIONED_HOSTS,
- StringUtils.join(calculateDecommissionedNodes(service, slaveCompType), ','));
-
if (serviceName.equals(Service.Type.HBASE.name())) {
commandParams.put(DECOM_EXCLUDED_HOSTS, StringUtils.join(listOfExcludedHosts, ','));
if ((isDrainOnlyRequest != null) && isDrainOnlyRequest.equals("true")) {
@@ -1077,17 +1072,6 @@ public class AmbariCustomCommandExecutionHelper {
}
}
- private Set<String> calculateDecommissionedNodes(Service service, String slaveCompType) throws AmbariException {
- Set<String> decommissionedHostsSet = new HashSet<>();
- ServiceComponent serviceComponent = service.getServiceComponent(slaveCompType);
- for (ServiceComponentHost serviceComponentHost : serviceComponent.getServiceComponentHosts().values()) {
- if (serviceComponentHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) {
- decommissionedHostsSet.add(serviceComponentHost.getHostName());
- }
- }
- return decommissionedHostsSet;
- }
-
private StringBuilder getReadableDecommissionCommandDetail(
ActionExecutionContext actionExecutionContext, Set<String> includedHosts,
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index f023b3a..1130e3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -58,8 +58,10 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.ActionExecutionContext;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent;
import org.apache.ambari.server.topology.TopologyManager;
import org.apache.commons.lang.StringUtils;
@@ -90,6 +92,8 @@ public class StageUtils {
protected static final String RACKS = "all_racks";
protected static final String IPV4_ADDRESSES = "all_ipv4_ips";
+ private static Map<String, String> decommissionedToClusterInfoKeyMap =
+ new HashMap<>();
private volatile static Gson gson;
@Inject
@@ -145,6 +149,12 @@ public class StageUtils {
StageUtils.configuration = configuration;
}
+ static {
+ decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts");
+ decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts");
+ decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts");
+ decommissionedToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "decom_hbase_rs_hosts");
+ }
public static String getActionId(long requestId, long stageId) {
return requestId + "-" + stageId;
@@ -291,6 +301,8 @@ public class StageUtils {
String roleName = getClusterHostInfoKey(componentName);
+ String decomRoleName = decommissionedToClusterInfoKeyMap.get(componentName);
+
for (String hostName : serviceComponent.getServiceComponentHosts().keySet()) {
if (roleName != null) {
@@ -305,6 +317,22 @@ public class StageUtils {
//Add index of host to current host role
hostsForComponentsHost.add(hostIndex);
}
+
+ if (decomRoleName != null) {
+ ServiceComponentHost scHost = serviceComponent.getServiceComponentHost(hostName);
+ if (scHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) {
+ SortedSet<Integer> hostsForComponentsHost = hostRolesInfo.get(decomRoleName);
+
+ if (hostsForComponentsHost == null) {
+ hostsForComponentsHost = new TreeSet<>();
+ hostRolesInfo.put(decomRoleName, hostsForComponentsHost);
+ }
+
+ int hostIndex = hostsList.indexOf(hostName);
+ //Add index of host to current host role
+ hostsForComponentsHost.add(hostIndex);
+ }
+ }
}
}
}
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 55476f9..1a6b2a8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -166,10 +166,7 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
falcon_user = config['configurations']['falcon-env']['falcon_user']
#exclude file
-if 'all_decommissioned_hosts' in config['commandParams']:
- hdfs_exclude_file = config['commandParams']['all_decommissioned_hosts'].split(",")
-else:
- hdfs_exclude_file = []
+hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
slave_hosts = default("/clusterHostInfo/datanode_hosts", [])
include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 8df14e1..0de72e3 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -310,10 +310,7 @@ yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc
user_group = config['configurations']['cluster-env']['user_group']
#exclude file
-if 'all_decommissioned_hosts' in config['commandParams']:
- exclude_hosts = config['commandParams']['all_decommissioned_hosts'].split(",")
-else:
- exclude_hosts = []
+exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index bf21825..af78e29 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -2840,7 +2840,6 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.HBASE_MASTER, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
- Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts"));
assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
// RS stops
@@ -2871,7 +2870,6 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
command = storedTasks.get(0);
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
- Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertTrue("DECOMMISSION".equals(command.getCustomCommandName()));
Assert.assertTrue(("DECOMMISSION, Excluded: " + host2).equals(command.getCommandDetail()));
cmdParams = command.getExecutionCommandWrapper().getExecutionCommand().getCommandParams();
@@ -5940,7 +5938,6 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.NAMENODE, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
- Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
// Decommission the other datanode
@@ -5970,10 +5967,6 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState());
Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
- Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host1));
- Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host2));
- Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host1+","+host2) ||
- execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host2+","+host1));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
// Recommission the other datanode (while adding NameNode HA)
@@ -6027,7 +6020,6 @@ public class AmbariManagementControllerTest {
|| !cmdParams.get("update_files_only").equals("true")) {
countRefresh++;
}
- Assert.assertEquals("", cmdParams.get("all_decommissioned_hosts"));
}
Assert.assertEquals(2, countRefresh);
@@ -6773,7 +6765,6 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.NAMENODE, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
- Assert.assertEquals(host1, execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
index e901d62..8f3fe8e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
@@ -548,6 +548,10 @@ public class StageUtilsTest extends EasyMockSupport {
}
assertEquals(expectedPingPorts, reindexedPorts);
+ assertTrue(info.containsKey("decom_tt_hosts"));
+ Set<String> decommissionedHosts = info.get("decom_tt_hosts");
+ assertEquals(2, decommissionedHosts.toString().split(",").length);
+
// check server hostname field
assertTrue(info.containsKey(StageUtils.AMBARI_SERVER_HOST));
Set<String> serverHost = info.get(StageUtils.AMBARI_SERVER_HOST);
--
To stop receiving notification emails like this one, please contact
avijayan@apache.org.