You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/24 19:07:56 UTC
[34/50] [abbrv] hadoop git commit: YARN-7491. Make sure AM is not
scheduled on an opportunistic container. Contributed by Haibo Chen.
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/979420f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/979420f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/979420f4
Branch: refs/heads/YARN-1011
Commit: 979420f4189a0e4ad371a69013f7f7ebd1febf55
Parents: 353bdbc
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 16:39:11 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++-
.../yarn/server/utils/TestBuilderUtils.java | 54 +++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 73 ++++++++++++++++----
4 files changed, 129 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index ff88225..a507e8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -351,15 +351,28 @@ public class BuilderUtils {
return priority;
}
+ public static ResourceRequest newAMResourceRequest(Priority priority,
+ String hostName, Resource capability) {
+ return newResourceRequest(priority, hostName, capability, 1,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+ }
+
public static ResourceRequest newResourceRequest(Priority priority,
String hostName, Resource capability, int numContainers) {
+ return newResourceRequest(priority, hostName, capability,
+ numContainers, ExecutionTypeRequest.newInstance());
+ }
+
+ public static ResourceRequest newResourceRequest(Priority priority,
+ String hostName, Resource capability, int numContainers,
+ ExecutionTypeRequest executionTypeRequest) {
ResourceRequest request = recordFactory
.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
- request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+ request.setExecutionTypeRequest(executionTypeRequest);
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+ @Test
+ public void testNewAMResourceRequest() {
+ final Priority priority = Priority.newInstance(0);
+ final String hostName = "ResourceA";
+ final Resource capacity = Resource.newInstance(1024, 1);
+
+ ResourceRequest resourceRequest =
+ BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+ Assert.assertEquals("unexpected priority", priority,
+ resourceRequest.getPriority());
+ Assert.assertEquals("unexpected host name", hostName,
+ resourceRequest.getResourceName());
+ Assert.assertEquals("unexpected capacity", capacity,
+ resourceRequest.getCapability());
+ Assert.assertEquals("unexpected number of containers", 1,
+ resourceRequest.getNumContainers());
+ Assert.assertEquals("unexpected execution type",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+ resourceRequest.getExecutionTypeRequest());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index d0f2ce6..6ea55ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -516,8 +516,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (amReqs == null || amReqs.isEmpty()) {
if (submissionContext.getResource() != null) {
amReqs = Collections.singletonList(BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1));
+ .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource()));
} else {
throw new InvalidResourceRequestException("Invalid resource request, "
+ "no resources requested");
@@ -548,7 +548,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
for (ResourceRequest amReq : amReqs) {
amReq.setCapability(anyReq.getCapability());
amReq.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
amReq.setNumContainers(1);
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979420f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 0bd5372..8617dc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -839,15 +839,45 @@ public class TestAppManager{
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequest(
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build());
List<ResourceRequest> reqs = new ArrayList<>();
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack", Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack/node", Resources.createResource(1025), 1, true));
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack/node")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
// getAMContainerResourceRequest uses the first entry of
// getAMContainerResourceRequests
@@ -869,9 +899,15 @@ public class TestAppManager{
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
ResourceRequest req =
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, true);
- req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression(RMNodeLabelsManager.NO_LABEL)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build();
asContext.setAMContainerResourceRequest(ResourceRequest.clone(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
@@ -892,9 +928,16 @@ public class TestAppManager{
// setResource
Assert.assertEquals(Collections.singletonList(
- ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, Resources.createResource(1024), 1, true,
- "")),
+ ResourceRequest.newBuilder()
+ .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression("")
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()),
app.getAMResourceRequests());
}
@@ -936,7 +979,7 @@ public class TestAppManager{
for (ResourceRequest req : reqs) {
req.setCapability(anyReq.getCapability());
req.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
req.setNumContainers(1);
req.setPriority(Priority.newInstance(0));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org