You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2016/03/08 20:06:46 UTC
[45/50] [abbrv] hadoop git commit: YARN-4335. Allow ResourceRequests
to specify ExecutionType of a request ask (kkaranasos via asuresh)
YARN-4335. Allow ResourceRequests to specify ExecutionType of a request ask (kkaranasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed6a17d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed6a17d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed6a17d7
Branch: refs/heads/yarn-2877
Commit: ed6a17d75ef3bf476716f20aabba1c620aada30b
Parents: 743a99f
Author: Arun Suresh <as...@apache.org>
Authored: Thu Jan 7 15:18:59 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Tue Mar 8 10:30:26 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES-yarn-2877.txt | 15 +++++++
.../yarn/api/records/ResourceRequest.java | 41 ++++++++++++++++++-
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/client/api/AMRMClient.java | 43 ++++++++++++++++++--
.../records/impl/pb/ResourceRequestPBImpl.java | 21 ++++++++++
hadoop-yarn-project/pom.xml | 1 +
6 files changed, 118 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed6a17d7/hadoop-yarn-project/CHANGES-yarn-2877.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES-yarn-2877.txt b/hadoop-yarn-project/CHANGES-yarn-2877.txt
new file mode 100644
index 0000000..d29ff0f
--- /dev/null
+++ b/hadoop-yarn-project/CHANGES-yarn-2877.txt
@@ -0,0 +1,15 @@
+Hadoop YARN Change Log
+
+Hadoop YARN Change Log for YARN-2877
+
+yarn-2877 distributed scheduling (Unreleased)
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ YARN-4335. Allow ResourceRequests to specify ExecutionType of a request
+ ask (kkaranasos via asuresh)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed6a17d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 790120a..8c1fd8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -79,6 +79,15 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
public static ResourceRequest newInstance(Priority priority, String hostName,
Resource capability, int numContainers, boolean relaxLocality,
String labelExpression) {
+ return newInstance(priority, hostName, capability, numContainers,
+ relaxLocality, labelExpression, ExecutionType.GUARANTEED);
+ }
+
+ @Public
+ @Stable
+ public static ResourceRequest newInstance(Priority priority, String hostName,
+ Resource capability, int numContainers, boolean relaxLocality, String
+ labelExpression, ExecutionType execType) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
@@ -86,6 +95,7 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
request.setNumContainers(numContainers);
request.setRelaxLocality(relaxLocality);
request.setNodeLabelExpression(labelExpression);
+ request.setExecutionType(execType);
return request;
}
@@ -221,7 +231,28 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
@Public
@Stable
public abstract boolean getRelaxLocality();
-
+
+ /**
+ * Set the <code>ExecutionType</code> of the requested container.
+ *
+ * @param execType
+ * ExecutionType of the requested container
+ */
+ @Public
+ @Stable
+ public abstract void setExecutionType(ExecutionType execType);
+
+ /**
+ * Get whether locality relaxation is enabled with this
+ * <code>ResourceRequest</code>. Defaults to true.
+ *
+ * @return whether locality relaxation is enabled with this
+ * <code>ResourceRequest</code>.
+ */
+ @Public
+ @Stable
+ public abstract ExecutionType getExecutionType();
+
/**
* <p>For a request at a network hierarchy level, set whether locality can be relaxed
* to that level and beyond.<p>
@@ -322,6 +353,14 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
return false;
} else if (!priority.equals(other.getPriority()))
return false;
+ ExecutionType executionType = getExecutionType();
+ if (executionType == null) {
+ if (other.getExecutionType() != null) {
+ return false;
+ }
+ } else if (executionType != other.getExecutionType()) {
+ return false;
+ }
if (getNodeLabelExpression() == null) {
if (other.getNodeLabelExpression() != null) {
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed6a17d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 9392efd..3ec5693 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -303,6 +303,7 @@ message ResourceRequestProto {
optional int32 num_containers = 4;
optional bool relax_locality = 5 [default = true];
optional string node_label_expression = 6;
+ optional ExecutionTypeProto executionType = 7 [default = GUARANTEED];
}
enum AMCommandProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed6a17d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index e0bf2d3..73c2725 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -108,6 +109,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
final Priority priority;
final boolean relaxLocality;
final String nodeLabelsExpression;
+ final ExecutionType executionType;
/**
* Instantiates a {@link ContainerRequest} with the given constraints and
@@ -152,6 +154,33 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
String[] racks, Priority priority, boolean relaxLocality) {
this(capability, nodes, racks, priority, relaxLocality, null);
}
+
+ /**
+ * Instantiates a {@link ContainerRequest} with the given constraints.
+ *
+ * @param capability
+ * The {@link Resource} to be requested for each container.
+ * @param nodes
+ * Any hosts to request that the containers are placed on.
+ * @param racks
+ * Any racks to request that the containers are placed on. The
+ * racks corresponding to any hosts requested will be automatically
+ * added to this list.
+ * @param priority
+ * The priority at which to request the containers. Higher
+ * priorities have lower numerical values.
+ * @param relaxLocality
+ * If true, containers for this request may be assigned on hosts
+ * and racks other than the ones explicitly requested.
+ * @param nodeLabelsExpression
+ * Set node labels to allocate resource, now we only support
+ * asking for only a single node label
+ */
+ public ContainerRequest(Resource capability, String[] nodes, String[] racks,
+ Priority priority, boolean relaxLocality, String nodeLabelsExpression) {
+ this(capability, nodes, racks, priority, relaxLocality, null,
+ ExecutionType.GUARANTEED);
+ }
/**
* Instantiates a {@link ContainerRequest} with the given constraints.
@@ -173,10 +202,12 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
* @param nodeLabelsExpression
* Set node labels to allocate resource, now we only support
* asking for only a single node label
+ * @param executionType
+ * Set the execution type of the container request.
*/
- public ContainerRequest(Resource capability, String[] nodes,
- String[] racks, Priority priority, boolean relaxLocality,
- String nodeLabelsExpression) {
+ public ContainerRequest(Resource capability, String[] nodes, String[] racks,
+ Priority priority, boolean relaxLocality, String nodeLabelsExpression,
+ ExecutionType executionType) {
// Validate request
Preconditions.checkArgument(capability != null,
"The Resource to be requested for each container " +
@@ -194,6 +225,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
this.priority = priority;
this.relaxLocality = relaxLocality;
this.nodeLabelsExpression = nodeLabelsExpression;
+ this.executionType = executionType;
}
public Resource getCapability() {
@@ -220,10 +252,15 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
return nodeLabelsExpression;
}
+ public ExecutionType getExecutionType() {
+ return executionType;
+ }
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("Capability[").append(capability).append("]");
sb.append("Priority[").append(priority).append("]");
+ sb.append("ExecutionType[").append(executionType).append("]");
return sb.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed6a17d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index fc09cef..53ae2cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -206,4 +207,24 @@ public class ResourceRequestPBImpl extends ResourceRequest {
}
builder.setNodeLabelExpression(nodeLabelExpression);
}
+
+ @Override
+ public ExecutionType getExecutionType() {
+ ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasExecutionType()) {
+ return null;
+ }
+ return ProtoUtils.convertFromProtoFormat(p.getExecutionType());
+ }
+
+ @Override
+ public void setExecutionType(ExecutionType execType) {
+ maybeInitBuilder();
+ if (execType == null) {
+ builder.clearExecutionType();
+ return;
+ }
+ builder.setExecutionType(ProtoUtils.convertToProtoFormat(execType));
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed6a17d7/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index e21e0b8..f451c48 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -110,6 +110,7 @@
<configuration>
<excludes>
<exclude>CHANGES.txt</exclude>
+ <exclude>CHANGES-yarn-2877.txt</exclude>
</excludes>
</configuration>
</plugin>