You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/01/29 05:11:02 UTC
[44/50] [abbrv] hadoop git commit: YARN-7491. Make sure AM is not
scheduled on an opportunistic container. Contributed by Haibo Chen.
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee49eb92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee49eb92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee49eb92
Branch: refs/heads/YARN-1011
Commit: ee49eb925a4c6f808855dd089b95cf0346d2de06
Parents: a2bb70c
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:23:37 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++-
.../yarn/server/utils/TestBuilderUtils.java | 54 +++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 73 ++++++++++++++++----
4 files changed, 130 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 63333ac..6abc617 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -349,15 +349,28 @@ public class BuilderUtils {
return priority;
}
+ public static ResourceRequest newAMResourceRequest(Priority priority,
+ String hostName, Resource capability) {
+ return newResourceRequest(priority, hostName, capability, 1,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+ }
+
public static ResourceRequest newResourceRequest(Priority priority,
String hostName, Resource capability, int numContainers) {
+ return newResourceRequest(priority, hostName, capability,
+ numContainers, ExecutionTypeRequest.newInstance());
+ }
+
+ public static ResourceRequest newResourceRequest(Priority priority,
+ String hostName, Resource capability, int numContainers,
+ ExecutionTypeRequest executionTypeRequest) {
ResourceRequest request = recordFactory
.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
- request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+ request.setExecutionTypeRequest(executionTypeRequest);
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+ @Test
+ public void testNewAMResourceRequest() {
+ final Priority priority = Priority.newInstance(0);
+ final String hostName = "ResourceA";
+ final Resource capacity = Resource.newInstance(1024, 1);
+
+ ResourceRequest resourceRequest =
+ BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+ Assert.assertEquals("unexpected priority", priority,
+ resourceRequest.getPriority());
+ Assert.assertEquals("unexpected host name", hostName,
+ resourceRequest.getResourceName());
+ Assert.assertEquals("unexpected capacity", capacity,
+ resourceRequest.getCapability());
+ Assert.assertEquals("unexpected number of containers", 1,
+ resourceRequest.getNumContainers());
+ Assert.assertEquals("unexpected execution type",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+ resourceRequest.getExecutionTypeRequest());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 2983077..1e05fec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -488,8 +488,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (amReqs == null || amReqs.isEmpty()) {
if (submissionContext.getResource() != null) {
amReqs = Collections.singletonList(BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1));
+ .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource()));
} else {
throw new InvalidResourceRequestException("Invalid resource request, "
+ "no resources requested");
@@ -520,7 +520,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
for (ResourceRequest amReq : amReqs) {
amReq.setCapability(anyReq.getCapability());
amReq.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
amReq.setNumContainers(1);
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee49eb92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 7c2ce64..140c15f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -691,15 +691,45 @@ public class TestAppManager{
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequest(
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build());
List<ResourceRequest> reqs = new ArrayList<>();
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack", Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack/node", Resources.createResource(1025), 1, true));
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack/node")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
// getAMContainerResourceRequest uses the first entry of
// getAMContainerResourceRequests
@@ -720,8 +750,14 @@ public class TestAppManager{
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
ResourceRequest req =
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, true);
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build();
asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
@@ -742,8 +778,16 @@ public class TestAppManager{
RMApp app = testRMAppSubmit();
// setResource
Assert.assertEquals(Collections.singletonList(
- ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")),
+ ResourceRequest.newBuilder()
+ .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression("")
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()),
app.getAMResourceRequests());
}
@@ -783,7 +827,7 @@ public class TestAppManager{
for (ResourceRequest req : reqs) {
req.setCapability(anyReq.getCapability());
req.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
req.setNumContainers(1);
req.setPriority(Priority.newInstance(0));
}
@@ -1100,7 +1144,8 @@ public class TestAppManager{
req.getNodeLabelExpression() != null
? new String(req.getNodeLabelExpression()) : null,
ExecutionTypeRequest.newInstance(
- req.getExecutionTypeRequest().getExecutionType()));
+ req.getExecutionTypeRequest().getExecutionType(),
+ req.getExecutionTypeRequest().getEnforceExecutionType()));
}
private static List<ResourceRequest> cloneResourceRequests(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org