You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2017/12/18 02:13:37 UTC
hadoop git commit: YARN-7617. Add a flag in distributed shell to
automatically PROMOTE opportunistic containers to guaranteed once they are
started. Contributed by Weiwei Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 5e81f32d1 -> 928964102
YARN-7617. Add a flag in distributed shell to automatically PROMOTE opportunistic containers to guaranteed once they are started. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92896410
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92896410
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92896410
Branch: refs/heads/trunk
Commit: 928964102029e96406f5482e8900802f38164501
Parents: 5e81f32
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Dec 18 10:07:16 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Mon Dec 18 10:07:16 2017 +0800
----------------------------------------------------------------------
.../distributedshell/ApplicationMaster.java | 49 +++++++++++++++++++-
.../applications/distributedshell/Client.java | 11 +++++
.../site/markdown/OpportunisticContainers.md.vm | 2 +-
3 files changed, 59 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92896410/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 926de50..b3fa0ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -93,6 +93,8 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -247,6 +249,8 @@ public class ApplicationMaster {
// Execution type of the containers.
// Default GUARANTEED.
private ExecutionType containerType = ExecutionType.GUARANTEED;
+ // Whether to automatically promote opportunistic containers.
+ private boolean autoPromoteContainers = false;
// Resource profile for the container
private String containerResourceProfile = "";
@@ -420,6 +424,9 @@ public class ApplicationMaster {
"Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption("container_type", true,
"Container execution type, GUARANTEED or OPPORTUNISTIC");
+ opts.addOption("promote_opportunistic_after_start", false,
+ "Flag to indicate whether to automatically promote opportunistic"
+ + " containers to guaranteed.");
opts.addOption("container_memory", true,
"Amount of memory in MB to be requested to run the shell command");
opts.addOption("container_vcores", true,
@@ -576,6 +583,9 @@ public class ApplicationMaster {
}
containerType = ExecutionType.valueOf(containerTypeStr);
}
+ if (cliParser.hasOption("promote_opportunistic_after_start")) {
+ autoPromoteContainers = true;
+ }
containerMemory = Integer.parseInt(cliParser.getOptionValue(
"container_memory", "-1"));
containerVirtualCores = Integer.parseInt(cliParser.getOptionValue(
@@ -977,7 +987,24 @@ public class ApplicationMaster {
@Override
public void onContainersUpdated(
- List<UpdatedContainer> containers) {}
+ List<UpdatedContainer> containers) {
+ for (UpdatedContainer container : containers) {
+ LOG.info("Container {} updated, updateType={}, resource={}, "
+ + "execType={}",
+ container.getContainer().getId(),
+ container.getUpdateType().toString(),
+ container.getContainer().getResource().toString(),
+ container.getContainer().getExecutionType());
+
+ // TODO Remove this line with finalized updateContainer API.
+ // Currently nm client needs to notify the NM to update container
+ // execution type via NMClient#updateContainerResource() or
+ // NMClientAsync#updateContainerResourceAsync() when
+ // auto-update.containers is disabled, but this API is
+ // under evolving and will need to be replaced by a proper new API.
+ nmClientAsync.updateContainerResourceAsync(container.getContainer());
+ }
+ }
@Override
public void onShutdownRequest() {
@@ -1004,7 +1031,7 @@ public class ApplicationMaster {
}
@VisibleForTesting
- static class NMCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+ class NMCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
private ConcurrentMap<ContainerId, Container> containers =
new ConcurrentHashMap<ContainerId, Container>();
@@ -1033,6 +1060,24 @@ public class ApplicationMaster {
LOG.debug("Container Status: id=" + containerId + ", status=" +
containerStatus);
}
+
+ // If promote_opportunistic_after_start is set, automatically promote
+ // opportunistic containers to guaranteed.
+ if (autoPromoteContainers) {
+ if (containerStatus.getState() == ContainerState.RUNNING) {
+ Container container = containers.get(containerId);
+ if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
+ // Promote container
+ LOG.info("Promoting container {} to {}", container.getId(),
+ container.getExecutionType());
+ UpdateContainerRequest updateRequest = UpdateContainerRequest
+ .newInstance(container.getVersion(), container.getId(),
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null,
+ ExecutionType.GUARANTEED);
+ amRMClient.requestContainerUpdate(container, updateRequest);
+ }
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92896410/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 16bf0fd..e299acc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -175,6 +175,8 @@ public class Client {
private String nodeLabelExpression = null;
// Container type, default GUARANTEED.
private ExecutionType containerType = ExecutionType.GUARANTEED;
+ // Whether to auto promote opportunistic containers
+ private boolean autoPromoteContainers = false;
// log4j.properties file
// if available, add to local resources and set into classpath
@@ -292,6 +294,9 @@ public class Client {
opts.addOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command");
opts.addOption("container_resource_profile", true, "Resource profile for the shell command");
opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed");
+ opts.addOption("promote_opportunistic_after_start", false,
+ "Flag to indicate whether to automatically promote opportunistic"
+ + " containers to guaranteed.");
opts.addOption("log_properties", true, "log4j.properties file");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application attempts." +
@@ -449,6 +454,9 @@ public class Client {
}
containerType = ExecutionType.valueOf(containerTypeStr);
}
+ if (cliParser.hasOption("promote_opportunistic_after_start")) {
+ autoPromoteContainers = true;
+ }
containerMemory =
Integer.parseInt(cliParser.getOptionValue("container_memory", "-1"));
containerVirtualCores =
@@ -759,6 +767,9 @@ public class Client {
if (containerType != null) {
vargs.add("--container_type " + String.valueOf(containerType));
}
+ if (autoPromoteContainers) {
+ vargs.add("--promote_opportunistic_after_start");
+ }
if (containerMemory > 0) {
vargs.add("--container_memory " + String.valueOf(containerMemory));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92896410/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md.vm
index 7882b87..f1c75ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md.vm
@@ -83,7 +83,7 @@ Another sample job is the distributed shell, it allows us to run a given shell c
$ yarn org.apache.hadoop.yarn.applications.distributedshell.Client -jar share/hadoop/yarn/hadoop-yarn-applications-distributedshell-${project.version}.jar.jar -shell_command sleep -shell_args 10 -num_containers 10 -container_type OPPORTUNISTIC
```
-By change the value of `container_type` to `OPPORTUNISTIC` or `GUARANTEED` in the above command, we can specify the tasks to be running in opportunistic or guaranteed containers. The default type is `GUARANTEED`.
+By change the value of `container_type` to `OPPORTUNISTIC` or `GUARANTEED` in the above command, we can specify the tasks to be running in opportunistic or guaranteed containers. The default type is `GUARANTEED`. By adding flag `-promote_opportunistic_after_start` to the above command, application master will attempt to promote all opportunistic containers to guaranteed once they are started.
$H3 Opportunistic Containers in Web UI
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org