You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2020/04/17 17:39:33 UTC
[hadoop] branch branch-3.2 updated: YARN-9954. Configurable max
application tags and max tag length. Contributed by Bilwa S T
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new d1af4e0 YARN-9954. Configurable max application tags and max tag length. Contributed by Bilwa S T
d1af4e0 is described below
commit d1af4e0fae842d46c29562444ebb16905f2caf0d
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Fri Apr 17 09:43:36 2020 -0700
YARN-9954. Configurable max application tags and max tag length. Contributed by Bilwa S T
(cherry picked from commit 49ae9b2137dddaff1b579586e593a24da21a9e7d)
---
.../api/records/ApplicationSubmissionContext.java | 4 +-
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 16 +++++++-
.../pb/ApplicationSubmissionContextPBImpl.java | 20 ----------
.../src/main/resources/yarn-default.xml | 13 +++++++
.../server/resourcemanager/ClientRMService.java | 27 +++++++++++++
.../resourcemanager/TestClientRMService.java | 45 ++++++++++++++++++++++
6 files changed, 101 insertions(+), 24 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index d2adfdc..1d4ffc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -424,9 +424,9 @@ public abstract class ApplicationSubmissionContext {
/**
* Set tags for the application. A maximum of
- * {@link YarnConfiguration#APPLICATION_MAX_TAGS} are allowed
+ * {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
* per application. Each tag can be at most
- * {@link YarnConfiguration#APPLICATION_MAX_TAG_LENGTH}
+ * {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
* characters, and can contain only ASCII characters.
*
* @param tags tags to set
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d3fa7ca..5484eae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -92,10 +92,10 @@ public class YarnConfiguration extends Configuration {
YARN_SITE_CONFIGURATION_FILE,
CORE_SITE_CONFIGURATION_FILE));
- @Evolving
+ @Deprecated
public static final int APPLICATION_MAX_TAGS = 10;
- @Evolving
+ @Deprecated
public static final int APPLICATION_MAX_TAG_LENGTH = 100;
static {
@@ -206,6 +206,18 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_ADDRESS =
"0.0.0.0:" + DEFAULT_RM_PORT;
+ /**Max number of application tags.*/
+ public static final String RM_APPLICATION_MAX_TAGS = RM_PREFIX
+ + "application.max-tags";
+
+ public static final int DEFAULT_RM_APPLICATION_MAX_TAGS = 10;
+
+ /**Max length of each application tag.*/
+ public static final String RM_APPLICATION_MAX_TAG_LENGTH = RM_PREFIX
+ + "application.max-tag.length";
+
+ public static final int DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH = 100;
+
public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS =
RM_PREFIX + "application-master-service.processors";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
index b30224e..01f0983 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProtoOrBuilder;
@@ -277,24 +276,6 @@ extends ApplicationSubmissionContext {
builder.setApplicationType((applicationType));
}
- private void checkTags(Set<String> tags) {
- if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) {
- throw new IllegalArgumentException("Too many applicationTags, a maximum of only "
- + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!");
- }
- for (String tag : tags) {
- if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) {
- throw new IllegalArgumentException("Tag " + tag + " is too long, " +
- "maximum allowed length of a tag is " +
- YarnConfiguration.APPLICATION_MAX_TAG_LENGTH);
- }
- if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) {
- throw new IllegalArgumentException("A tag can only have ASCII " +
- "characters! Invalid tag - " + tag);
- }
- }
- }
-
@Override
public synchronized void setApplicationTags(Set<String> tags) {
maybeInitBuilder();
@@ -303,7 +284,6 @@ extends ApplicationSubmissionContext {
this.applicationTags = null;
return;
}
- checkTags(tags);
// Convert applicationTags to lower case and add
this.applicationTags = new HashSet<String>();
for (String tag : tags) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index eba6c03..4a208f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -4155,4 +4155,17 @@
<value></value>
</property>
+ <property>
+ <description>Max number of application tags set by user in ApplicationSubmissionContext
+ while submitting application</description>
+ <name>yarn.resourcemanager.application.max-tags</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>Max length of each application tag set by user in ApplicationSubmissionContext
+ while submitting application.</description>
+ <name>yarn.resourcemanager.application.max-tag.length</name>
+ <value>100</value>
+ </property>
</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index c7213bb..87e70b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -610,6 +610,8 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
+ checkTags(submissionContext.getApplicationTags());
+
if (timelineServiceV2Enabled) {
// Sanity check for flow run
String value = null;
@@ -752,6 +754,31 @@ public class ClientRMService extends AbstractService implements
return response;
}
+ private void checkTags(Set<String> tags) throws YarnException {
+ int appMaxTags = getConfig().getInt(
+ YarnConfiguration.RM_APPLICATION_MAX_TAGS,
+ YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAGS);
+ int appMaxTagLength = getConfig().getInt(
+ YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH,
+ YarnConfiguration.DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH);
+ if (tags.size() > appMaxTags) {
+ throw RPCUtil.getRemoteException(new IllegalArgumentException(
+ "Too many applicationTags, a maximum of only " + appMaxTags
+ + " are allowed!"));
+ }
+ for (String tag : tags) {
+ if (tag.length() > appMaxTagLength) {
+ throw RPCUtil.getRemoteException(
+ new IllegalArgumentException("Tag " + tag + " is too long, "
+ + "maximum allowed length of a tag is " + appMaxTagLength));
+ }
+ if (!org.apache.commons.lang3.StringUtils.isAsciiPrintable(tag)) {
+ throw RPCUtil.getRemoteException(new IllegalArgumentException(
+ "A tag can only have ASCII " + "characters! Invalid tag - " + tag));
+ }
+ }
+ }
+
@SuppressWarnings("unchecked")
@Override
public KillApplicationResponse forceKillApplication(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 07c6383..d5a9978 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -593,6 +593,51 @@ public class TestClientRMService {
}
@Test
+ public void testApplicationTagsValidation() throws IOException {
+ YarnConfiguration conf = new YarnConfiguration();
+ int maxtags = 3, appMaxTagLength = 5;
+ conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAGS, maxtags);
+ conf.setInt(YarnConfiguration.RM_APPLICATION_MAX_TAG_LENGTH,
+ appMaxTagLength);
+ MockRM rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+
+ ClientRMService rmService = rm.getClientRMService();
+
+ List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3", "Tag4");
+ validateApplicationTag(rmService, tags,
+ "Too many applicationTags, a maximum of only " + maxtags
+ + " are allowed!");
+
+ tags = Arrays.asList("ApplicationTag1", "ApplicationTag2",
+ "ApplicationTag3");
+ // tags are converted to lowercase in
+ // ApplicationSubmissionContext#setApplicationTags
+ validateApplicationTag(rmService, tags,
+ "Tag applicationtag1 is too long, maximum allowed length of a tag is "
+ + appMaxTagLength);
+
+ tags = Arrays.asList("tãg1", "tag2#");
+ validateApplicationTag(rmService, tags,
+ "A tag can only have ASCII characters! Invalid tag - tãg1");
+ rm.close();
+ }
+
+ private void validateApplicationTag(ClientRMService rmService,
+ List<String> tags, String errorMsg) {
+ SubmitApplicationRequest submitRequest = mockSubmitAppRequest(
+ getApplicationId(101), MockApps.newAppName(), QUEUE_1,
+ new HashSet<String>(tags));
+ try {
+ rmService.submitApplication(submitRequest);
+ Assert.fail();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().contains(errorMsg));
+ }
+ }
+
+ @Test
public void testForceKillApplication() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(MockRM.ENABLE_WEBAPP, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org