You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by qu...@apache.org on 2022/03/23 09:42:31 UTC
[hadoop] branch trunk updated: YARN-11084. Introduce new config to specify AM default node-label when not specified. Contributed by Junfan Zhang.
This is an automated email from the ASF dual-hosted git repository.
quapaw pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 921267c YARN-11084. Introduce new config to specify AM default node-label when not specified. Contributed by Junfan Zhang.
921267c is described below
commit 921267ca3103763b4237195029cdcb06e8c2232d
Author: 9uapaw <gy...@gmail.com>
AuthorDate: Wed Mar 23 08:54:42 2022 +0100
YARN-11084. Introduce new config to specify AM default node-label when not specified. Contributed by Junfan Zhang.
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++
.../yarn/server/resourcemanager/RMAppManager.java | 15 ++++++++---
.../server/resourcemanager/TestAppManager.java | 31 ++++++++++++++++++++++
.../src/site/markdown/NodeLabel.md | 8 ++++++
4 files changed, 54 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e61b541..df01a12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -4409,6 +4409,9 @@ public class YarnConfiguration extends Configuration {
private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX
+ "node-labels.";
+ public static final String AM_DEFAULT_NODE_LABEL =
+ RM_NODE_LABELS_PREFIX + "am.default-node-label-expression";
+
public static final String RM_NODE_LABELS_PROVIDER_CONFIG =
RM_NODE_LABELS_PREFIX + "provider";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 928eeb1..5bedef4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -83,6 +83,9 @@ import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture;
import org.apache.hadoop.yarn.util.StringHelper;
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static org.apache.commons.lang.StringUtils.isNotEmpty;
+
/**
* This class manages the list of applications for the resource manager.
*/
@@ -106,6 +109,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private boolean timelineServiceV2Enabled;
private boolean nodeLabelsEnabled;
private Set<String> exclusiveEnforcedPartitions;
+ private String amDefaultNodeLabel;
private static final String USER_ID_PREFIX = "userid=";
@@ -134,6 +138,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.areNodeLabelsEnabled(rmContext.getYarnConfiguration());
this.exclusiveEnforcedPartitions = YarnConfiguration
.getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration());
+ this.amDefaultNodeLabel = conf
+ .get(YarnConfiguration.AM_DEFAULT_NODE_LABEL, null);
}
/**
@@ -622,9 +628,12 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
}
// set label expression for AM ANY request if not set
- if (null == anyReq.getNodeLabelExpression()) {
- anyReq.setNodeLabelExpression(submissionContext
- .getNodeLabelExpression());
+ if (isEmpty(anyReq.getNodeLabelExpression())) {
+ if (isNotEmpty(amDefaultNodeLabel)) {
+ anyReq.setNodeLabelExpression(amDefaultNodeLabel);
+ } else {
+ anyReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression());
+ }
}
// Put ANY request at the front
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8c73a0d..f9ecc85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -730,6 +730,33 @@ public class TestAppManager extends AppManagerTestBase{
}
@Test
+ public void testRMAppSubmitAMContainerWithNoLabelByRMDefaultAMNodeLabel() throws Exception {
+ List<ResourceRequest> reqs = new ArrayList<>();
+ ResourceRequest anyReq = ResourceRequest.newInstance(
+ Priority.newInstance(1),
+ ResourceRequest.ANY, Resources.createResource(1024), 1, false, null,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ reqs.add(anyReq);
+ asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+ asContext.setNodeLabelExpression("fixed");
+
+ Configuration conf = new Configuration(false);
+ String defaultAMNodeLabel = "core";
+ conf.set(YarnConfiguration.AM_DEFAULT_NODE_LABEL, defaultAMNodeLabel);
+
+ when(mockDefaultQueueInfo.getAccessibleNodeLabels()).thenReturn(
+ new HashSet<String>() {{ add("core"); }});
+
+ TestRMAppManager newAppMonitor = createAppManager(rmContext, conf);
+ newAppMonitor.submitApplication(asContext, "test");
+
+ RMApp app = rmContext.getRMApps().get(appId);
+ waitUntilEventProcessed();
+ Assert.assertEquals(defaultAMNodeLabel,
+ app.getAMResourceRequests().get(0).getNodeLabelExpression());
+ }
+
+ @Test
public void testRMAppSubmitResource() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
@@ -836,6 +863,10 @@ public class TestAppManager extends AppManagerTestBase{
private RMApp testRMAppSubmit() throws Exception {
appMonitor.submitApplication(asContext, "test");
+ return waitUntilEventProcessed();
+ }
+
+ private RMApp waitUntilEventProcessed() throws InterruptedException {
RMApp app = rmContext.getRMApps().get(appId);
Assert.assertNotNull("app is null", app);
Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
index 32545f3..9c39e99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
@@ -170,6 +170,14 @@ Applications can use following Java APIs to specify node label to request
* `ResourceRequest.setNodeLabelExpression(..)` to set node label expression for individual resource requests. This can overwrite node label expression set in ApplicationSubmissionContext
* Specify `setAMContainerResourceRequest.setNodeLabelExpression` in `ApplicationSubmissionContext` to indicate expected node label for application master container.
+__Default AM node-label Configuration__
+
+Property | Value
+----- | ------
+yarn.resourcemanager.node-labels.am.default-node-label-expression | Overwrites default-node-label-expression only for the ApplicationMaster container. It is disabled by default.
+
+
+
Monitoring
----------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org