You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/02/24 20:54:17 UTC
[hadoop] branch trunk updated: YARN-10135. FS-CS converter tool:
issue warning on dynamic auto-create mapping rules. Contributed by Peter
Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 72fa9c7 YARN-10135. FS-CS converter tool: issue warning on dynamic auto-create mapping rules. Contributed by Peter Bacsko
72fa9c7 is described below
commit 72fa9c7f1b262839b88533a61e8f2ffbbfd15a1d
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Mon Feb 24 21:54:07 2020 +0100
YARN-10135. FS-CS converter tool: issue warning on dynamic auto-create mapping rules. Contributed by Peter Bacsko
---
.../converter/FSConfigToCSConfigRuleHandler.java | 22 +++++++++++++++++++++-
.../fair/converter/QueuePlacementConverter.java | 20 ++++++++++++++------
.../converter/TestQueuePlacementConverter.java | 10 +++++++---
3 files changed, 42 insertions(+), 10 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
index 3edbd62..f4526cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
@@ -76,6 +76,9 @@ public class FSConfigToCSConfigRuleHandler {
public static final String FAIR_AS_DRF =
"fairAsDrf.action";
+ public static final String MAPPED_DYNAMIC_QUEUE =
+ "mappedDynamicQueue.action";
+
@VisibleForTesting
enum RuleAction {
WARNING,
@@ -122,6 +125,7 @@ public class FSConfigToCSConfigRuleHandler {
setActionForProperty(RESERVATION_SYSTEM);
setActionForProperty(QUEUE_AUTO_CREATE);
setActionForProperty(FAIR_AS_DRF);
+ setActionForProperty(MAPPED_DYNAMIC_QUEUE);
}
public void handleMaxCapacityPercentage(String queueName) {
@@ -180,7 +184,8 @@ public class FSConfigToCSConfigRuleHandler {
handle(QUEUE_AUTO_CREATE,
null,
format(
- "Placement rules: queue auto-create is not supported (type: %s)",
+ "Placement rules: queue auto-create is not supported (type: %s),"
+ + " please configure auto-create-child-queue property manually",
placementRule));
}
@@ -192,6 +197,21 @@ public class FSConfigToCSConfigRuleHandler {
queueName));
}
+ public void handleDynamicMappedQueue(String mapping, boolean create) {
+ String msg = "Mapping rule %s is dynamic - this might cause inconsistent"
+ + " behaviour compared to FS.";
+
+ if (create) {
+ msg += " Also, setting auto-create-child-queue=true is"
+ + " necessary, because the create flag was set to true on the"
+ + " original placement rule.";
+ }
+
+ handle(MAPPED_DYNAMIC_QUEUE,
+ null,
+ format(msg, mapping));
+ }
+
private void handle(String actionName, String fsSetting, String message) {
RuleAction action = actions.get(actionName);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
index c47dd76..bfd1b9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
@@ -57,7 +57,7 @@ class QueuePlacementConverter {
// nested rule
if (userRule.getParentRule() != null) {
- handleNestedRule(mapping, userRule);
+ handleNestedRule(mapping, userRule, ruleHandler);
} else {
if (!userAsDefaultQueue) {
if (mapping.length() > 0) {
@@ -102,20 +102,28 @@ class QueuePlacementConverter {
}
private void handleNestedRule(StringBuilder mapping,
- UserPlacementRule userRule) {
+ UserPlacementRule userRule, FSConfigToCSConfigRuleHandler ruleHandler) {
PlacementRule pr = userRule.getParentRule();
if (mapping.length() > 0) {
mapping.append(RULE_SEPARATOR);
}
if (pr instanceof PrimaryGroupPlacementRule) {
- mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER);
+ String mappingString = "u:" + USER + ":" + PRIMARY_GROUP + "." + USER;
+ ruleHandler.handleDynamicMappedQueue(mappingString,
+ ((PrimaryGroupPlacementRule) pr).getCreateFlag());
+ mapping.append(mappingString);
} else if (pr instanceof SecondaryGroupExistingPlacementRule) {
+ String mappingString = "u:" + USER + ":" + SECONDARY_GROUP + "." + USER;
+ ruleHandler.handleDynamicMappedQueue(mappingString,
+ ((SecondaryGroupExistingPlacementRule) pr).getCreateFlag());
mapping.append("u:" + USER + ":" + SECONDARY_GROUP + "." + USER);
} else if (pr instanceof DefaultPlacementRule) {
DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr;
- mapping.append("u:" + USER + ":")
- .append(defaultRule.defaultQueueName)
- .append("." + USER);
+ String mappingString =
+ "u:" + USER + ":" + defaultRule.defaultQueueName + "." + USER;
+ ruleHandler.handleDynamicMappedQueue(mappingString,
+ defaultRule.getCreateFlag());
+ mapping.append(mappingString);
} else {
throw new UnsupportedOperationException("Unsupported nested rule: "
+ pr.getClass().getCanonicalName());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
index 289d35b..1a644bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
@@ -19,6 +19,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
@@ -180,7 +181,8 @@ public class TestQueuePlacementConverter {
Map<String, String> properties = convert(false);
verifyMapping(properties, "u:%user:%primary_group.%user");
- verifyZeroInteractions(ruleHandler);
+ verify(ruleHandler).handleDynamicMappedQueue(
+ eq("u:%user:%primary_group.%user"), eq(false));
}
@Test
@@ -194,7 +196,8 @@ public class TestQueuePlacementConverter {
Map<String, String> properties = convert(false);
verifyMapping(properties, "u:%user:%secondary_group.%user");
- verifyZeroInteractions(ruleHandler);
+ verify(ruleHandler).handleDynamicMappedQueue(
+ eq("u:%user:%secondary_group.%user"), eq(false));
}
@Test
@@ -209,7 +212,8 @@ public class TestQueuePlacementConverter {
Map<String, String> properties = convert(false);
verifyMapping(properties, "u:%user:abc.%user");
- verifyZeroInteractions(ruleHandler);
+ verify(ruleHandler).handleDynamicMappedQueue(
+ eq("u:%user:abc.%user"), eq(false));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org