You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/07/24 15:26:46 UTC
[20/50] [abbrv] ambari git commit: AMBARI-21345 Add host doesn't
fully add a node when include/exclude files are used (dsen)
AMBARI-21345 Add host doesn't fully add a node when include/exclude files are used (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cc412e66
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cc412e66
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cc412e66
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: cc412e66156d5a887a725015537dcb75b0caf986
Parents: 93fe848
Author: Dmytro Sen <ds...@apache.org>
Authored: Mon Jul 17 13:36:58 2017 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Mon Jul 17 13:36:58 2017 +0300
----------------------------------------------------------------------
.../AmbariCustomCommandExecutionHelper.java | 14 ++-
.../AmbariManagementControllerImpl.java | 121 ++++++++++++++-----
.../internal/HostResourceProvider.java | 1 +
.../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 6 +
.../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 8 ++
.../2.1.0.2.0/package/scripts/hdfs_namenode.py | 25 +++-
.../2.1.0.2.0/package/scripts/params_linux.py | 9 +-
.../2.1.0.2.0/package/scripts/params_windows.py | 7 ++
.../package/templates/include_hosts_list.j2 | 21 ++++
.../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 8 ++
.../3.0.0.3.0/package/scripts/hdfs_namenode.py | 17 ++-
.../3.0.0.3.0/package/scripts/params_linux.py | 8 +-
.../3.0.0.3.0/package/scripts/params_windows.py | 7 ++
.../package/templates/include_hosts_list.j2 | 21 ++++
.../YARN/2.1.0.2.0/configuration/yarn-site.xml | 6 +
.../2.1.0.2.0/package/scripts/params_linux.py | 12 +-
.../2.1.0.2.0/package/scripts/params_windows.py | 10 +-
.../package/scripts/resourcemanager.py | 18 ++-
.../package/templates/include_hosts_list.j2 | 21 ++++
.../YARN/3.0.0.3.0/configuration/yarn-site.xml | 6 +
.../3.0.0.3.0/package/scripts/params_linux.py | 11 +-
.../3.0.0.3.0/package/scripts/params_windows.py | 10 +-
.../package/scripts/resourcemanager.py | 18 ++-
.../package/templates/include_hosts_list.j2 | 21 ++++
.../services/HDFS/configuration/hdfs-site.xml | 6 +
.../HDFS/package/scripts/hdfs_namenode.py | 12 +-
.../0.8/services/HDFS/package/scripts/params.py | 11 +-
.../package/templates/include_hosts_list.j2 | 21 ++++
.../services/YARN/configuration/yarn-site.xml | 6 +
.../0.8/services/YARN/package/scripts/params.py | 10 +-
.../YARN/package/scripts/resourcemanager.py | 9 +-
.../package/templates/include_hosts_list.j2 | 21 ++++
.../services/YARN/configuration/yarn-site.xml | 6 +
.../services/YARN/package/scripts/params.py | 9 +-
.../YARN/package/scripts/resourcemanager.py | 9 +-
.../package/templates/exclude_hosts_list.j2 | 21 ++++
.../package/templates/include_hosts_list.j2 | 21 ++++
.../services/YARN/configuration/yarn-site.xml | 6 +
.../services/YARN/package/scripts/params.py | 10 +-
.../YARN/package/scripts/resourcemanager.py | 9 +-
.../package/templates/include_hosts_list.j2 | 21 ++++
.../AmbariManagementControllerTest.java | 8 +-
.../python/stacks/2.0.6/HDFS/test_namenode.py | 2 +-
.../stacks/2.0.6/configs/altfs_plus_hdfs.json | 2 +-
.../python/stacks/2.0.6/configs/default.json | 2 +-
.../2.0.6/configs/default_ams_embedded.json | 2 +-
.../2.0.6/configs/default_hive_nn_ha.json | 2 +-
.../2.0.6/configs/default_hive_nn_ha_2.json | 2 +-
.../2.0.6/configs/default_hive_non_hdfs.json | 2 +-
.../2.0.6/configs/default_no_install.json | 2 +-
.../2.0.6/configs/default_oozie_mysql.json | 2 +-
.../default_update_exclude_file_only.json | 2 +-
.../2.0.6/configs/default_with_bucket.json | 2 +-
.../python/stacks/2.0.6/configs/flume_22.json | 2 +-
.../python/stacks/2.0.6/configs/flume_only.json | 2 +-
.../stacks/2.0.6/configs/hbase_no_phx.json | 2 +-
.../stacks/2.0.6/configs/hbase_with_phx.json | 2 +-
.../2.0.6/configs/oozie_existing_sqla.json | 2 +-
.../stacks/2.0.6/configs/repository_file.json | 2 +-
.../python/stacks/2.0.6/configs/secured.json | 2 +-
.../test/python/stacks/2.3/configs/ats_1_5.json | 2 +-
.../python/stacks/2.5/configs/hsi_default.json | 2 +-
.../2.5/configs/hsi_default_for_restart.json | 2 +-
.../test/python/stacks/2.5/configs/hsi_ha.json | 2 +-
.../services/YARN/configuration/yarn-site.xml | 6 +
.../YARN/package/scripts/params_linux.py | 9 +-
.../YARN/package/scripts/params_windows.py | 10 +-
.../YARN/package/scripts/resourcemanager.py | 18 ++-
.../package/templates/include_hosts_list.j2 | 21 ++++
69 files changed, 638 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 28aa4e4..aeb5a9c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -137,7 +137,7 @@ public class AmbariCustomCommandExecutionHelper {
public final static String DECOM_EXCLUDED_HOSTS = "excluded_hosts";
public final static String DECOM_SLAVE_COMPONENT = "slave_type";
public final static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only";
- public final static String UPDATE_EXCLUDE_FILE_ONLY = "update_exclude_file_only";
+ public final static String UPDATE_FILES_ONLY = "update_files_only";
private final static String ALIGN_MAINTENANCE_STATE = "align_maintenance_state";
@@ -910,9 +910,9 @@ public class AmbariCustomCommandExecutionHelper {
@Override
public boolean shouldHostBeRemoved(final String hostname)
throws AmbariException {
- //Get UPDATE_EXCLUDE_FILE_ONLY parameter as string
+ //Get UPDATE_FILES_ONLY parameter as string
String upd_excl_file_only_str = actionExecutionContext.getParameters()
- .get(UPDATE_EXCLUDE_FILE_ONLY);
+ .get(UPDATE_FILES_ONLY);
String decom_incl_hosts_str = actionExecutionContext.getParameters()
.get(DECOM_INCLUDED_HOSTS);
@@ -986,15 +986,17 @@ public class AmbariCustomCommandExecutionHelper {
listOfExcludedHosts.add(sch.getHostName());
if (alignMtnState) {
sch.setMaintenanceState(MaintenanceState.ON);
+ LOG.info("marking Maintenance=ON on " + sch.getHostName());
}
- LOG.info("Decommissioning " + slaveCompType + " and marking Maintenance=ON on " + sch.getHostName());
+ LOG.info("Decommissioning " + slaveCompType + " on " + sch.getHostName());
}
if (filteredIncludedHosts.contains(sch.getHostName())) {
sch.setComponentAdminState(HostComponentAdminState.INSERVICE);
if (alignMtnState) {
sch.setMaintenanceState(MaintenanceState.OFF);
+ LOG.info("marking Maintenance=OFF on " + sch.getHostName());
}
- LOG.info("Recommissioning " + slaveCompType + " and marking Maintenance=OFF on " + sch.getHostName());
+ LOG.info("Recommissioning " + slaveCompType + " on " + sch.getHostName());
}
}
@@ -1048,7 +1050,7 @@ public class AmbariCustomCommandExecutionHelper {
}
if (!serviceName.equals(Service.Type.HBASE.name()) || hostName.equals(primaryCandidate)) {
- commandParams.put(UPDATE_EXCLUDE_FILE_ONLY, "false");
+ commandParams.put(UPDATE_FILES_ONLY, "false");
addCustomCommandAction(commandContext, commandFilter, stage, commandParams, commandDetail.toString(), null);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 5639dc1..433ed56 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -42,6 +42,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.UNLIMITED
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
+import static org.apache.ambari.server.controller.AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom;
import java.io.File;
import java.io.FileNotFoundException;
@@ -3334,17 +3335,49 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
}
+ Map<String, String> serviceMasterForDecommissionMap = new HashMap<>();
for (Map<State, List<ServiceComponentHost>> stateScHostMap :
changedScHosts.values()) {
for (Entry<State, List<ServiceComponentHost>> entry :
stateScHostMap.entrySet()) {
State newState = entry.getKey();
for (ServiceComponentHost sch : entry.getValue()) {
+ String componentName = sch.getServiceComponentName();
+ //Create map for include/exclude files refresh
+ if (masterToSlaveMappingForDecom.containsValue(componentName) &&
+ sch.getState() == State.INIT && newState == State.INSTALLED) {
+ String serviceName = sch.getServiceName();
+ String masterComponentName = null;
+ for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) {
+ if (entrySet.getValue().equals(componentName)) {
+ masterComponentName = entrySet.getKey();
+ }
+ }
+ try {
+ Service s = cluster.getService(serviceName);
+ //Filter services whose masters are not started
+ if (s.getServiceComponent(masterComponentName).getDesiredState() == State.STARTED) {
+ serviceMasterForDecommissionMap.put(serviceName, masterComponentName);
+ } else {
+ LOG.info(String.format("Not adding %s service from include/exclude files refresh map because it's master is not started", serviceName));
+ }
+ } catch (AmbariException e) {
+ LOG.error("Exception during INIT masters cleanup : ", e);
+ }
+ }
+
+ //actually set the new state
sch.setDesiredState(newState);
}
}
}
+ try {
+ createAndExecuteRefreshIncludeExcludeFilesActionForMasters(serviceMasterForDecommissionMap, cluster.getClusterName());
+ } catch (AmbariException e) {
+ LOG.error("Exception during refresh include exclude files action : ", e);
+ }
+
if (ignoredScHosts != null) {
for (ServiceComponentHost scHost : ignoredScHosts) {
scHost.setDesiredState(scHost.getState());
@@ -3582,18 +3615,39 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
}
+ Map<String, Map<String, String>> clusterServiceMasterForDecommissionMap = new HashMap<>();
for (Entry<ServiceComponent, Set<ServiceComponentHost>> entry : safeToRemoveSCHs.entrySet()) {
for (ServiceComponentHost componentHost : entry.getValue()) {
try {
deleteHostComponent(entry.getKey(), componentHost);
deleteStatusMetaData.addDeletedKey(componentHost.getHostName() + "/" + componentHost.getServiceComponentName());
-
+ //create cluster-master-service map to update all include/exclude files in one action
+ String componentName = componentHost.getServiceComponentName();
+ if (masterToSlaveMappingForDecom.containsValue(componentName)) {
+ String masterComponentName = null;
+ for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) {
+ if (entrySet.getValue().equals(componentName)) {
+ masterComponentName = entrySet.getKey();
+ }
+ }
+ if (clusterServiceMasterForDecommissionMap.containsKey(componentHost.getClusterName())) {
+ clusterServiceMasterForDecommissionMap.get(componentHost.getClusterName()).put(componentHost.getServiceName(), masterComponentName);
+ } else {
+ Map<String, String> tempMap = new HashMap<>();
+ tempMap.put(componentHost.getServiceName(), masterComponentName);
+ clusterServiceMasterForDecommissionMap.put(componentHost.getClusterName(), tempMap);
+ }
+ }
} catch (Exception ex) {
deleteStatusMetaData.addException(componentHost.getHostName() + "/" + componentHost.getServiceComponentName(), ex);
}
}
}
+ for (String cluster : clusterServiceMasterForDecommissionMap.keySet()) {
+ createAndExecuteRefreshIncludeExcludeFilesActionForMasters(clusterServiceMasterForDecommissionMap.get(cluster), cluster);
+ }
+
//Do not break behavior for existing clients where delete request contains only 1 host component.
//Response for these requests will have empty body with appropriate error code.
if (deleteStatusMetaData.getDeletedKeys().size() + deleteStatusMetaData.getExceptionForKeys().size() == 1) {
@@ -3616,7 +3670,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
private void deleteHostComponent(ServiceComponent serviceComponent, ServiceComponentHost componentHost) throws AmbariException {
- String included_hostname = componentHost.getHostName();
String serviceName = serviceComponent.getServiceName();
String master_component_name = null;
String slave_component_name = componentHost.getServiceComponentName();
@@ -3624,37 +3677,15 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
State slaveState = componentHost.getState();
//Delete hostcomponents
serviceComponent.deleteServiceComponentHosts(componentHost.getHostName());
- // If deleted hostcomponents support decomission and were decommited and stopped
- if (AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom.containsValue(slave_component_name)
+ // If deleted hostcomponents support decomission and were decommited and stopped or in unknown state
+ if (masterToSlaveMappingForDecom.containsValue(slave_component_name)
&& desiredAdminState.equals(HostComponentAdminState.DECOMMISSIONED)
- && slaveState.equals(State.INSTALLED)) {
-
- for (Entry<String, String> entrySet : AmbariCustomCommandExecutionHelper.masterToSlaveMappingForDecom.entrySet()) {
+ && (slaveState.equals(State.INSTALLED) || slaveState.equals(State.UNKNOWN))) {
+ for (Entry<String, String> entrySet : masterToSlaveMappingForDecom.entrySet()) {
if (entrySet.getValue().equals(slave_component_name)) {
master_component_name = entrySet.getKey();
}
}
- //Clear exclud file or draining list except HBASE
- if (!serviceName.equals(Service.Type.HBASE.toString())) {
- HashMap<String, String> requestProperties = new HashMap<>();
- requestProperties.put("context", "Remove host " +
- included_hostname + " from exclude file");
- requestProperties.put("exclusive", "true");
- HashMap<String, String> params = new HashMap<>();
- params.put("included_hosts", included_hostname);
- params.put("slave_type", slave_component_name);
- params.put(AmbariCustomCommandExecutionHelper.UPDATE_EXCLUDE_FILE_ONLY, "true");
-
- //Create filter for RECOMISSION command
- RequestResourceFilter resourceFilter
- = new RequestResourceFilter(serviceName, master_component_name, null);
- //Create request for RECOMISSION command
- ExecuteActionRequest actionRequest = new ExecuteActionRequest(
- serviceComponent.getClusterName(), AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null,
- Collections.singletonList(resourceFilter), null, params, true);
- //Send request
- createAction(actionRequest, requestProperties);
- }
//Mark master component as needed to restart for remove host info from components UI
Cluster cluster = clusters.getCluster(serviceComponent.getClusterName());
@@ -3695,6 +3726,40 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
}
+ /**
+ * Creates and triggers an action to update include and exclude files for the master components depending on current cluster topology and components state
+ * @param serviceMasterMap
+ * @param clusterName
+ * @throws AmbariException
+ */
+ private void createAndExecuteRefreshIncludeExcludeFilesActionForMasters(Map<String, String> serviceMasterMap, String clusterName) throws AmbariException {
+ //Clear include/exclude files or draining list except HBASE
+ serviceMasterMap.remove(Service.Type.HBASE.toString());
+ //exit if empty
+ if (serviceMasterMap.isEmpty()) {
+ return;
+ }
+ LOG.debug("Refresh include/exclude files action will be executed for " + serviceMasterMap);
+ HashMap<String, String> requestProperties = new HashMap<>();
+ requestProperties.put("context", "Update Include and Exclude Files for " + serviceMasterMap.keySet().toString());
+ requestProperties.put("exclusive", "true");
+ HashMap<String, String> params = new HashMap<>();
+ params.put(AmbariCustomCommandExecutionHelper.UPDATE_FILES_ONLY, "false");
+
+ //Create filter for command
+ List<RequestResourceFilter> resourceFilters = new ArrayList<>(serviceMasterMap.size());
+ for (String serviceName : serviceMasterMap.keySet()) {
+ resourceFilters.add(new RequestResourceFilter(serviceName, serviceMasterMap.get(serviceName), null));
+ }
+
+ //Create request for command
+ ExecuteActionRequest actionRequest = new ExecuteActionRequest(
+ clusterName, AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null,
+ resourceFilters, null, params, true);
+ //Send action
+ createAction(actionRequest, requestProperties);
+ }
+
@Override
public void deleteMembers(java.util.Set<MemberRequest> requests) throws AmbariException {
for (MemberRequest request : requests) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
index 4e2944f..8ef42ba 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java
@@ -946,6 +946,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider {
throw new AmbariException(reason.toString());
}
} else {
+// TODO why host with all components stopped can't be deleted? This functional is implemented and only this validation stops the request.
if (!componentsToRemove.isEmpty()) {
StringBuilder reason = new StringBuilder("Cannot remove host ")
.append(hostName)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
index aad2db0..4eab367 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml
@@ -99,6 +99,12 @@
excluded.</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if dfs.hosts is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
<!--
<property>
<name>dfs.hosts</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index 15fda67..e054209 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -170,6 +170,14 @@ def hdfs(component=None):
owner=params.hdfs_user,
mode="f",
)
+
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ mode="f",
+ )
+ pass
if params.service_map.has_key(component):
service_name = params.service_map[component]
ServiceConfig(service_name,
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
index 7226d22..cac6e9c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py
@@ -108,6 +108,14 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None,
group=params.user_group
)
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ pass
+
if do_format and not params.hdfs_namenode_format_disabled:
format_namenode()
pass
@@ -437,7 +445,15 @@ def decommission():
group=user_group
)
- if not params.update_exclude_file_only:
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ pass
+
+ if not params.update_files_only:
Execute(nn_kinit_cmd,
user=hdfs_user
)
@@ -464,6 +480,13 @@ def decommission():
owner=hdfs_user
)
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user
+ )
+ pass
+
if params.dfs_ha_enabled:
# due to a bug in hdfs, refreshNodes will not run on both namenodes so we
# need to execute each command scoped to a particular namenode
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index a9fc179..2854a00 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -163,7 +163,13 @@ falcon_user = config['configurations']['falcon-env']['falcon_user']
#exclude file
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
+hdfs_include_file = None
+manage_include_files = default("/configurations/hdfs-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file))
+update_files_only = default("/commandParams/update_files_only",False)
command_phase = default("/commandParams/phase","")
klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None))
@@ -172,7 +178,6 @@ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executab
hostname = config["hostname"]
public_hostname = config["public_hostname"]
rm_host = default("/clusterHostInfo/rm_host", [])
-slave_hosts = default("/clusterHostInfo/slave_hosts", [])
oozie_servers = default("/clusterHostInfo/oozie_server", [])
hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
hive_server_host = default("/clusterHostInfo/hive_server_host", [])
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py
index 1e47c29..b3ac578 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_windows.py
@@ -43,6 +43,13 @@ dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir']
#decomission
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
+include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
+hdfs_include_file = None
+manage_include_files = default("/configurations/hdfs-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+ hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file))
+update_files_only = default("/commandParams/update_files_only",False)
# HDFS High Availability properties
dfs_ha_enabled = False
dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..115a8a4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in hdfs_include_file %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
index 15fda67..e054209 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
@@ -170,6 +170,14 @@ def hdfs(component=None):
owner=params.hdfs_user,
mode="f",
)
+
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ mode="f",
+ )
+ pass
if params.service_map.has_key(component):
service_name = params.service_map[component]
ServiceConfig(service_name,
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py
index 181b3c8..5a1f368 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs_namenode.py
@@ -436,7 +436,15 @@ def decommission():
group=user_group
)
- if not params.update_exclude_file_only:
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ pass
+
+ if not params.update_files_only:
Execute(nn_kinit_cmd,
user=hdfs_user
)
@@ -463,6 +471,13 @@ def decommission():
owner=hdfs_user
)
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user
+ )
+ pass
+
if params.dfs_ha_enabled:
# due to a bug in hdfs, refreshNodes will not run on both namenodes so we
# need to execute each command scoped to a particular namenode
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
index ad49d81..1581c2a 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
@@ -158,7 +158,13 @@ falcon_user = config['configurations']['falcon-env']['falcon_user']
#exclude file
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
+hdfs_include_file = None
+manage_include_files = default("/configurations/hdfs-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file))
+update_files_only = default("/commandParams/update_files_only", False)
command_phase = default("/commandParams/phase","")
klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None))
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py
index 70d95a6..6c492d8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_windows.py
@@ -43,6 +43,13 @@ dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir']
#decomission
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
+include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
+hdfs_include_file = None
+manage_include_files = default("/configurations/hdfs-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+ hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file))
+update_files_only = default("/commandParams/update_files_only",False)
# HDFS High Availability properties
dfs_ha_enabled = False
dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..115a8a4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in hdfs_include_file %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
index d0d0ede..a65b801 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml
@@ -408,6 +408,12 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>yarn.http.policy</name>
<value>HTTP_ONLY</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index f474a89..67931c6 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -241,11 +241,17 @@ user_group = config['configurations']['cluster-env']['user_group']
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
+
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
has_ats = not len(ats_host) == 0
-nm_hosts = default("/clusterHostInfo/nm_hosts", [])
-
# don't using len(nm_hosts) here, because check can take too much time on large clusters
number_of_nm = 1
@@ -345,7 +351,7 @@ HdfsResource = functools.partial(
immutable_paths = get_not_managed_resources(),
dfs_type = dfs_type
)
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+update_files_only = default("/commandParams/update_files_only",False)
mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py
index 52918d2e..c2a02d7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_windows.py
@@ -59,4 +59,12 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-update_exclude_file_only = config['commandParams']['update_exclude_file_only']
+
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
+update_files_only = default("/commandParams/update_files_only", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
index 7d024b1..b929af0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
@@ -91,7 +91,14 @@ class ResourcemanagerWindows(Resourcemanager):
mode="f"
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ mode="f"
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd, user=yarn_user)
@@ -159,7 +166,14 @@ class ResourcemanagerDefault(Resourcemanager):
group=user_group
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ group=user_group
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd,
environment= {'PATH' : params.execute_path },
user=yarn_user)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..42e33c0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in include_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
index 64e0bcb..2a69d35 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-site.xml
@@ -402,6 +402,12 @@
<on-ambari-upgrade add="false"/>
</property>
<property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>yarn.http.policy</name>
<value>HTTP_ONLY</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
index a05d259..68d17f0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
@@ -239,10 +239,17 @@ user_group = config['configurations']['cluster-env']['user_group']
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
+
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
has_ats = not len(ats_host) == 0
-nm_hosts = default("/clusterHostInfo/nm_hosts", [])
# don't using len(nm_hosts) here, because check can take too much time on large clusters
number_of_nm = 1
@@ -341,7 +348,7 @@ HdfsResource = functools.partial(
immutable_paths = get_not_managed_resources(),
dfs_type = dfs_type
)
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+update_files_only = default("/commandParams/update_files_only", False)
mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py
index 52918d2e..c2a02d7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_windows.py
@@ -59,4 +59,12 @@ hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
-update_exclude_file_only = config['commandParams']['update_exclude_file_only']
+
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
+update_files_only = default("/commandParams/update_files_only", False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
index f6d6315..961fe63 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
@@ -91,7 +91,14 @@ class ResourcemanagerWindows(Resourcemanager):
mode="f"
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ mode="f"
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd, user=yarn_user)
@@ -174,7 +181,14 @@ class ResourcemanagerDefault(Resourcemanager):
group=user_group
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ mode="f"
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd,
environment= {'PATH' : params.execute_path },
user=yarn_user)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..42e33c0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in include_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml
index 87684df..66d25cf 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/configuration/hdfs-site.xml
@@ -72,6 +72,12 @@
excluded.</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if dfs.hosts is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
<!--
<property>
<name>dfs.hosts</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py
index 6de7735..19751f6 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/hdfs_namenode.py
@@ -143,8 +143,16 @@ def decommission():
owner=hdfs_user,
group=user_group
)
-
- if not params.update_exclude_file_only:
+
+ if params.hdfs_include_file:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ pass
+
+ if not params.update_files_only:
Execute(nn_kinit_cmd,
user=hdfs_user
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
index 19e223c..9cf163a 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
@@ -55,13 +55,18 @@ falcon_user = config['configurations']['falcon-env']['falcon_user']
#exclude file
hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+include_file_path = default("/configurations/hdfs-site/dfs.hosts", None)
+hdfs_include_file = None
+manage_include_files = default("/configurations/hdfs-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ hdfs_include_file = list(set(slave_hosts) - set(hdfs_exclude_file))
+update_files_only = default("/commandParams/update_files_only",False)
kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
#hosts
hostname = config["hostname"]
rm_host = default("/clusterHostInfo/rm_host", [])
-slave_hosts = default("/clusterHostInfo/slave_hosts", [])
oozie_servers = default("/clusterHostInfo/oozie_server", [])
hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
hive_server_host = default("/clusterHostInfo/hive_server_host", [])
@@ -241,4 +246,4 @@ ttnode_heapsize = "1024m"
dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
-script_https_protocol = Script.get_force_https_protocol_name()
\ No newline at end of file
+script_https_protocol = Script.get_force_https_protocol_name()
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..115a8a4
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in hdfs_include_file %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml
index 8e9b8b1..3cb5add 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/configuration/yarn-site.xml
@@ -334,6 +334,12 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>yarn.timeline-service.enabled</name>
<value>true</value>
<description>Indicate to clients whether timeline service is enabled or not.
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py
index 33496cfe..87b5992 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/params.py
@@ -118,6 +118,14 @@ user_group = config['configurations']['cluster-env']['user_group']
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.exclude-path","/etc/hadoop/conf/yarn.exclude")
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
+
hostname = config['hostname']
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
@@ -162,7 +170,7 @@ HdfsDirectory = functools.partial(
kinit_path_local = kinit_path_local,
bin_dir = hadoop_bin_dir
)
-update_exclude_file_only = default("/commandParams/update_exclude_file_only",False)
+update_files_only = default("/commandParams/update_files_only",False)
mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py
index 4d40d68..8bd76bf 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/scripts/resourcemanager.py
@@ -89,7 +89,14 @@ class Resourcemanager(Script):
group=user_group
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ group=user_group
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd,
environment= {'PATH' : params.execute_path },
user=yarn_user)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..42e33c0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/YARN/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in include_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml
index 93a35cd..f2da835 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/configuration/yarn-site.xml
@@ -334,6 +334,12 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>yarn.timeline-service.enabled</name>
<value>true</value>
<description>Indicate to clients whether timeline service is enabled or not.
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py
index 5a7e508..16db0e4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/params.py
@@ -98,6 +98,13 @@ yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
#exclude file
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path']
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
hostname = config['hostname']
@@ -128,7 +135,7 @@ HdfsDirectory = functools.partial(
keytab = hdfs_user_keytab,
kinit_path_local = kinit_path_local
)
-update_exclude_file_only = config['commandParams']['update_exclude_file_only']
+update_files_only = default("/commandParams/update_files_only", False)
hadoop_bin = "/usr/lib/hadoop/sbin"
mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py
index a286ae3..f92938b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/scripts/resourcemanager.py
@@ -79,7 +79,14 @@ class Resourcemanager(Script):
group=user_group
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ group=user_group
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd,
user=yarn_user)
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2
new file mode 100644
index 0000000..c7ce416
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in exclude_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..42e33c0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/YARN/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in include_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml
index 93a35cd..f2da835 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/configuration/yarn-site.xml
@@ -334,6 +334,12 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>manage.include.files</name>
+ <value>false</value>
+ <description>If true Ambari will manage include file if yarn.resourcemanager.nodes.include-path is configured.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>yarn.timeline-service.enabled</name>
<value>true</value>
<description>Indicate to clients whether timeline service is enabled or not.
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py
index 5a7e508..bd188e9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/params.py
@@ -98,7 +98,13 @@ yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
#exclude file
exclude_hosts = default("/clusterHostInfo/decom_nm_hosts", [])
exclude_file_path = config['configurations']['yarn-site']['yarn.resourcemanager.nodes.exclude-path']
-
+nm_hosts = default("/clusterHostInfo/nm_hosts", [])
+#incude file
+include_file_path = default("/configurations/yarn-site/yarn.resourcemanager.nodes.include-path", None)
+include_hosts = None
+manage_include_files = default("/configurations/yarn-site/manage.include.files", False)
+if include_file_path and manage_include_files:
+ include_hosts = list(set(nm_hosts) - set(exclude_hosts))
hostname = config['hostname']
if security_enabled:
@@ -128,7 +134,7 @@ HdfsDirectory = functools.partial(
keytab = hdfs_user_keytab,
kinit_path_local = kinit_path_local
)
-update_exclude_file_only = config['commandParams']['update_exclude_file_only']
+update_files_only = default("/commandParams/update_files_only", False)
hadoop_bin = "/usr/lib/hadoop/sbin"
mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py
index a286ae3..f92938b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/resourcemanager.py
@@ -79,7 +79,14 @@ class Resourcemanager(Script):
group=user_group
)
- if params.update_exclude_file_only == False:
+ if params.include_hosts:
+ File(params.include_file_path,
+ content=Template("include_hosts_list.j2"),
+ owner=yarn_user,
+ group=user_group
+ )
+
+ if params.update_files_only == False:
Execute(yarn_refresh_cmd,
user=yarn_user)
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2
new file mode 100644
index 0000000..42e33c0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/include_hosts_list.j2
@@ -0,0 +1,21 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+{% for host in include_hosts %}
+{{host}}
+{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index b3a12f2..fdfca0f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -6028,8 +6028,8 @@ public class AmbariManagementControllerTest {
execCmd = storedTasks.get(0).getExecutionCommandWrapper
().getExecutionCommand();
Map<String, String> cmdParams = execCmd.getCommandParams();
- Assert.assertTrue(cmdParams.containsKey("update_exclude_file_only"));
- Assert.assertTrue(cmdParams.get("update_exclude_file_only").equals("false"));
+ Assert.assertTrue(cmdParams.containsKey("update_files_only"));
+ Assert.assertTrue(cmdParams.get("update_files_only").equals("false"));
Assert.assertNotNull(storedTasks);
Assert.assertEquals(1, storedTasks.size());
Assert.assertEquals(HostComponentAdminState.DECOMMISSIONED, scHost.getComponentAdminState());
@@ -6084,8 +6084,8 @@ public class AmbariManagementControllerTest {
Assert.assertTrue(hrc.getCommandDetail().contains(host1));
Assert.assertTrue(hrc.getCommandDetail().contains(host2));
cmdParams = hrc.getExecutionCommandWrapper().getExecutionCommand().getCommandParams();
- if(!cmdParams.containsKey("update_exclude_file_only")
- || !cmdParams.get("update_exclude_file_only").equals("true")) {
+ if(!cmdParams.containsKey("update_files_only")
+ || !cmdParams.get("update_files_only").equals("true")) {
countRefresh++;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index a7233c7..f0f2587 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -1041,7 +1041,7 @@ class TestNamenode(RMFTestCase):
bin_dir = '/usr/bin')
self.assertNoMoreResources()
- def test_decommission_update_exclude_file_only(self):
+ def test_decommission_update_files_only(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
classname = "NameNode",
command = "decommission",
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json b/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json
index ea00a37..f928073 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/altfs_plus_hdfs.json
@@ -33,7 +33,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index d267bc1..a16ec26 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -60,7 +60,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
index 71423c8..f3e8dc3 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
@@ -32,7 +32,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json
index 009ff6d..7b0f78d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha.json
@@ -35,7 +35,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json
index 2b078c3..01f0efc 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_nn_ha_2.json
@@ -35,7 +35,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json
index 571b737..0cbd322 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_hive_non_hdfs.json
@@ -33,7 +33,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json
index 7fdb449..cfcf5e1 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_no_install.json
@@ -35,7 +35,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
http://git-wip-us.apache.org/repos/asf/ambari/blob/cc412e66/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json
index 7378b68..7db73ab 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_oozie_mysql.json
@@ -36,7 +36,7 @@
"script": "scripts/service_check.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false",
+ "update_files_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
"properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],