You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2018/04/12 10:41:10 UTC
[ambari] branch trunk updated: [AMBARI-23493] Decommission Datanode
doesn't work (dsen)
This is an automated email from the ASF dual-hosted git repository.
dsen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 675ca9a [AMBARI-23493] Decommission Datanode doesn't work (dsen)
675ca9a is described below
commit 675ca9a64f70fae517aeefcadcbab137a239922a
Author: Dmytro Sen <ds...@apache.org>
AuthorDate: Thu Apr 12 11:58:51 2018 +0300
[AMBARI-23493] Decommission Datanode doesn't work (dsen)
---
.../AmbariCustomCommandExecutionHelper.java | 16 +++++++++++++
.../org/apache/ambari/server/utils/StageUtils.java | 28 ----------------------
.../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 5 +++-
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 5 +++-
.../controller/AmbariManagementControllerTest.java | 9 +++++++
.../apache/ambari/server/utils/StageUtilsTest.java | 4 ----
6 files changed, 33 insertions(+), 34 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 61b14c4..d396d12 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -129,6 +129,7 @@ public class AmbariCustomCommandExecutionHelper {
public final static String DECOM_INCLUDED_HOSTS = "included_hosts";
public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts";
+ public final static String ALL_DECOMMISSIONED_HOSTS = "all_decommissioned_hosts";
public final static String DECOM_SLAVE_COMPONENT = "slave_type";
public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only";
public final static String UPDATE_FILES_ONLY = "update_files_only";
@@ -1056,6 +1057,10 @@ public class AmbariCustomCommandExecutionHelper {
}
Map<String, String> commandParams = new HashMap<>();
+
+ commandParams.put(ALL_DECOMMISSIONED_HOSTS,
+ StringUtils.join(calculateDecommissionedNodes(service, slaveCompType), ','));
+
if (serviceName.equals(Service.Type.HBASE.name())) {
commandParams.put(DECOM_EXCLUDED_HOSTS, StringUtils.join(listOfExcludedHosts, ','));
if ((isDrainOnlyRequest != null) && isDrainOnlyRequest.equals("true")) {
@@ -1072,6 +1077,17 @@ public class AmbariCustomCommandExecutionHelper {
}
}
+ private Set<String> calculateDecommissionedNodes(Service service, String slaveCompType) throws AmbariException {
+ Set<String> decommissionedHostsSet = new HashSet<>();
+ ServiceComponent serviceComponent = service.getServiceComponent(slaveCompType);
+ for (ServiceComponentHost serviceComponentHost : serviceComponent.getServiceComponentHosts().values()) {
+ if (serviceComponentHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) {
+ decommissionedHostsSet.add(serviceComponentHost.getHostName());
+ }
+ }
+ return decommissionedHostsSet;
+ }
+
private StringBuilder getReadableDecommissionCommandDetail(
ActionExecutionContext actionExecutionContext, Set<String> includedHosts,
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index 1130e3d..f023b3a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -58,10 +58,8 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.ActionExecutionContext;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
-import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostInstallEvent;
import org.apache.ambari.server.topology.TopologyManager;
import org.apache.commons.lang.StringUtils;
@@ -92,8 +90,6 @@ public class StageUtils {
protected static final String RACKS = "all_racks";
protected static final String IPV4_ADDRESSES = "all_ipv4_ips";
- private static Map<String, String> decommissionedToClusterInfoKeyMap =
- new HashMap<>();
private volatile static Gson gson;
@Inject
@@ -149,12 +145,6 @@ public class StageUtils {
StageUtils.configuration = configuration;
}
- static {
- decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts");
- decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts");
- decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts");
- decommissionedToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "decom_hbase_rs_hosts");
- }
public static String getActionId(long requestId, long stageId) {
return requestId + "-" + stageId;
@@ -301,8 +291,6 @@ public class StageUtils {
String roleName = getClusterHostInfoKey(componentName);
- String decomRoleName = decommissionedToClusterInfoKeyMap.get(componentName);
-
for (String hostName : serviceComponent.getServiceComponentHosts().keySet()) {
if (roleName != null) {
@@ -317,22 +305,6 @@ public class StageUtils {
//Add index of host to current host role
hostsForComponentsHost.add(hostIndex);
}
-
- if (decomRoleName != null) {
- ServiceComponentHost scHost = serviceComponent.getServiceComponentHost(hostName);
- if (scHost.getComponentAdminState() == HostComponentAdminState.DECOMMISSIONED) {
- SortedSet<Integer> hostsForComponentsHost = hostRolesInfo.get(decomRoleName);
-
- if (hostsForComponentsHost == null) {
- hostsForComponentsHost = new TreeSet<>();
- hostRolesInfo.put(decomRoleName, hostsForComponentsHost);
- }
-
- int hostIndex = hostsList.indexOf(hostName);
- //Add index of host to current host role
- hostsForComponentsHost.add(hostIndex);
- }
- }
}
}
}
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 1a6b2a8..55476f9 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -166,7 +166,10 @@ hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
falcon_user = config['configurations']['falcon-env']['falcon_user']
#exclude file
-hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
+if 'all_decommissioned_hosts' in config['commandParams']:
+ hdfs_exclude_file = config['commandParams']['all_decommissioned_hosts'].split(",")
+else:
+ hdfs_exclude_file = []
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
slave_hosts = default("/clusterHostInfo/datanode_hosts", [])
include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 0de72e3..8df14e1 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -310,7 +310,10 @@ yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc
user_group = config['configurations']['cluster-env']['user_group']
#exclude file
-exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
+if 'all_decommissioned_hosts' in config['commandParams']:
+ exclude_hosts = config['commandParams']['all_decommissioned_hosts'].split(",")
+else:
+ exclude_hosts = []
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
rm_nodes_exclude_dir = os.path.dirname(exclude_file_path)
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index af78e29..bf21825 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -2840,6 +2840,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.HBASE_MASTER, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts"));
assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
// RS stops
@@ -2870,6 +2871,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
command = storedTasks.get(0);
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertTrue("DECOMMISSION".equals(command.getCustomCommandName()));
Assert.assertTrue(("DECOMMISSION, Excluded: " + host2).equals(command.getCommandDetail()));
cmdParams = command.getExecutionCommandWrapper().getExecutionCommand().getCommandParams();
@@ -5938,6 +5940,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.NAMENODE, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host2, execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
// Decommission the other datanode
@@ -5967,6 +5970,10 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState());
Assert.assertEquals(MaintenanceState.ON, scHost.getMaintenanceState());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
+ Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host1));
+ Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").contains(host2));
+ Assert.assertTrue(execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host1+","+host2) ||
+ execCmd.getCommandParams().get("all_decommissioned_hosts").equals(host2+","+host1));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
// Recommission the other datanode (while adding NameNode HA)
@@ -6020,6 +6027,7 @@ public class AmbariManagementControllerTest {
|| !cmdParams.get("update_files_only").equals("true")) {
countRefresh++;
}
+ Assert.assertEquals("", cmdParams.get("all_decommissioned_hosts"));
}
Assert.assertEquals(2, countRefresh);
@@ -6765,6 +6773,7 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(Role.NAMENODE, command.getRole());
Assert.assertEquals(RoleCommand.CUSTOM_COMMAND, command.getRoleCommand());
Assert.assertEquals("DECOMMISSION", execCmd.getCommandParams().get("custom_command"));
+ Assert.assertEquals(host1, execCmd.getCommandParams().get("all_decommissioned_hosts"));
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
index 8f3fe8e..e901d62 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
@@ -548,10 +548,6 @@ public class StageUtilsTest extends EasyMockSupport {
}
assertEquals(expectedPingPorts, reindexedPorts);
- assertTrue(info.containsKey("decom_tt_hosts"));
- Set<String> decommissionedHosts = info.get("decom_tt_hosts");
- assertEquals(2, decommissionedHosts.toString().split(",").length);
-
// check server hostname field
assertTrue(info.containsKey(StageUtils.AMBARI_SERVER_HOST));
Set<String> serverHost = info.get(StageUtils.AMBARI_SERVER_HOST);
--
To stop receiving notification emails like this one, please contact
dsen@apache.org.