You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/19 00:32:41 UTC
[02/39] hadoop git commit: YARN-4887. Add allocation request ID to
AM-RM protocol for identifying resource-requests explicitly. (Subru Krishnan
via wangda)
YARN-4887. Add allocation request ID to AM-RM protocol for identifying resource-requests explicitly. (Subru Krishnan via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eec835ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eec835ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eec835ec
Branch: refs/heads/HDFS-1312
Commit: eec835ec17f77bf713d90dbc2f77dea45f3ee660
Parents: e3ba9ad
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Jun 13 21:57:33 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Jun 13 21:57:33 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/records/Container.java | 45 +++++++++++++++++
.../yarn/api/records/ResourceRequest.java | 52 ++++++++++++++++++++
.../src/main/proto/yarn_protos.proto | 2 +
.../api/records/impl/pb/ContainerPBImpl.java | 14 ++++++
.../records/impl/pb/ResourceRequestPBImpl.java | 16 +++++-
5 files changed, 128 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
index 9a62935..707a71d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -189,4 +190,48 @@ public abstract class Container implements Comparable<Container> {
@Private
@Unstable
public abstract void setExecutionType(ExecutionType executionType);
+
+ /**
+ * Get the optional <em>ID</em> corresponding to the original {@code
+ * ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by
+ * this allocated {@code Container}.
+ * <p>
+ * The scheduler may return multiple {@code AllocateResponse}s corresponding
+ * to the same ID as and when scheduler allocates {@code Container}s.
+ * <b>Applications</b> can continue to completely ignore the returned ID in
+ * the response and use the allocation for any of their outstanding requests.
+ * <p>
+ *
+ * @return the <em>ID</em> corresponding to the original allocation request
+ * which is satisfied by this allocation.
+ */
+ @Public
+ @Evolving
+ public long getAllocationRequestId() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Set the optional <em>ID</em> corresponding to the original {@code
+ * ResourceRequest{@link #setAllocationRequestId(long)}
+ * etAllocationRequestId()}}s which is satisfied by this allocated {@code
+ * Container}.
+ * <p>
+ * The scheduler may return multiple {@code AllocateResponse}s corresponding
+ * to the same ID as and when scheduler allocates {@code Container}s.
+ * <b>Applications</b> can continue to completely ignore the returned ID in
+ * the response and use the allocation for any of their outstanding requests.
+ * If the ID is not set, scheduler will continue to work as previously and all
+ * allocated {@code Container}(s) will have the default ID, -1.
+ * <p>
+ *
+ * @param allocationRequestID the <em>ID</em> corresponding to the original
+ * allocation request which is satisfied by this
+ * allocation.
+ */
+ @Private
+ @Evolving
+ public void setAllocationRequestId(long allocationRequestID) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index fbe7e58..50a7619 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -312,6 +312,58 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
@Public
@Evolving
public abstract void setNodeLabelExpression(String nodelabelExpression);
+
+ /**
+ * Get the optional <em>ID</em> corresponding to this allocation request. This
+ * ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ * application</b>. The allocated {@code Container}(s) received as part of the
+ * {@code AllocateResponse} response will have the ID corresponding to the
+ * original {@code ResourceRequest} for which the RM made the allocation.
+ * <p>
+ * The scheduler may return multiple {@code AllocateResponse}s corresponding
+ * to the same ID as and when scheduler allocates {@code Container}(s).
+ * <b>Applications</b> can continue to completely ignore the returned ID in
+ * the response and use the allocation for any of their outstanding requests.
+ * <p>
+ * If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ * a specific ID, they can simply cancel the corresponding {@code
+ * ResourceRequest} and submit a new one afresh.
+ *
+ * @return the <em>ID</em> corresponding to this allocation request.
+ */
+ @Public
+ @Evolving
+ public long getAllocationRequestId() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Set the optional <em>ID</em> corresponding to this allocation request. This
+ * ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ * application</b>. The allocated {@code Container}(s) received as part of the
+ * {@code AllocateResponse} response will have the ID corresponding to the
+ * original {@code ResourceRequest} for which the RM made the allocation.
+ * <p>
+ * The scheduler may return multiple {@code AllocateResponse}s corresponding
+ * to the same ID as and when scheduler allocates {@code Container}(s).
+ * <b>Applications</b> can continue to completely ignore the returned ID in
+ * the response and use the allocation for any of their outstanding requests.
+ * <p>
+ * If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ * a specific ID, they can simply cancel the corresponding {@code
+ * ResourceRequest} and submit a new one afresh.
+ * <p>
+ * If the ID is not set, scheduler will continue to work as previously and all
+ * allocated {@code Container}(s) will have the default ID, -1.
+ *
+ * @param allocationRequestID the <em>ID</em> corresponding to this allocation
+ * request.
+ */
+ @Public
+ @Evolving
+ public void setAllocationRequestId(long allocationRequestID) {
+ throw new UnsupportedOperationException();
+ }
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 0649f8e..c84f4e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -93,6 +93,7 @@ message ContainerProto {
optional PriorityProto priority = 5;
optional hadoop.common.TokenProto container_token = 6;
optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED];
+ optional int64 allocation_request_id = 8 [default = -1];
}
message ContainerReportProto {
@@ -302,6 +303,7 @@ message ResourceRequestProto {
optional bool relax_locality = 5 [default = true];
optional string node_label_expression = 6;
optional ExecutionTypeRequestProto execution_type_request = 7;
+ optional int64 allocation_request_id = 8 [default = -1];
}
message ExecutionTypeRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index bd2d937..91b3e5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -262,6 +262,18 @@ public class ContainerPBImpl extends Container {
builder.setExecutionType(convertToProtoFormat(executionType));
}
+ @Override
+ public long getAllocationRequestId() {
+ ContainerProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getAllocationRequestId());
+ }
+
+ @Override
+ public void setAllocationRequestId(long allocationRequestID) {
+ maybeInitBuilder();
+ builder.setAllocationRequestId(allocationRequestID);
+ }
+
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
return new ContainerIdPBImpl(p);
}
@@ -315,6 +327,8 @@ public class ContainerPBImpl extends Container {
StringBuilder sb = new StringBuilder();
sb.append("Container: [");
sb.append("ContainerId: ").append(getId()).append(", ");
+ sb.append("AllocationRequestId: ").append(getAllocationRequestId())
+ .append(", ");
sb.append("NodeId: ").append(getNodeId()).append(", ");
sb.append("NodeHttpAddress: ").append(getNodeHttpAddress()).append(", ");
sb.append("Resource: ").append(getResource()).append(", ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eec835ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index b0c4b97..9890296 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -192,6 +192,18 @@ public class ResourceRequestPBImpl extends ResourceRequest {
builder.setRelaxLocality(relaxLocality);
}
+ @Override
+ public long getAllocationRequestId() {
+ ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getAllocationRequestId());
+ }
+
+ @Override
+ public void setAllocationRequestId(long allocationRequestID) {
+ maybeInitBuilder();
+ builder.setAllocationRequestId(allocationRequestID);
+ }
+
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
return new PriorityPBImpl(p);
}
@@ -210,7 +222,9 @@ public class ResourceRequestPBImpl extends ResourceRequest {
@Override
public String toString() {
- return "{Priority: " + getPriority() + ", Capability: " + getCapability()
+ return "{AllocationRequestId: " + getAllocationRequestId()
+ + ", Priority: " + getPriority()
+ + ", Capability: " + getCapability()
+ ", # Containers: " + getNumContainers()
+ ", Location: " + getResourceName()
+ ", Relax Locality: " + getRelaxLocality()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org