You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2020/09/03 13:03:24 UTC
[hadoop] branch trunk updated: YARN-10372. Create MappingRule class
to represent each CS mapping rule. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 288dab7 YARN-10372. Create MappingRule class to represent each CS mapping rule. Contributed by Gergely Pollak
288dab7 is described below
commit 288dab75633b92f3104e61d3163e51d1f1263041
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Thu Sep 3 15:02:48 2020 +0200
YARN-10372. Create MappingRule class to represent each CS mapping rule. Contributed by Gergely Pollak
---
.../resourcemanager/placement/MappingRule.java | 136 ++++++++++++++++++++
.../resourcemanager/placement/TestMappingRule.java | 143 +++++++++++++++++++++
2 files changed, 279 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
new file mode 100644
index 0000000..e03be2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Mapping rule represents a single mapping setting defined by the user. All
+ * rules have matchers and actions. Matcher determine if a mapping rule applies
+ * to a given applicationSubmission, while action represent the course of action
+ * we need to take when a rule applies.
+ *
+ * MappingRules also support fallback actions, which will be evaluated when the
+ * main action fails due to any reason (Eg. trying to place to a queue which
+ * does not exist)
+ */
+public class MappingRule {
+ public static final String USER_MAPPING = "u";
+ public static final String GROUP_MAPPING = "g";
+ public static final String APPLICATION_MAPPING = "a";
+ private final MappingRuleMatcher matcher;
+ private final MappingRuleAction action;
+
+ public MappingRule(MappingRuleMatcher matcher, MappingRuleAction action) {
+ this.matcher = matcher;
+ this.action = action;
+ }
+
+ /**
+ * This method evaluates the rule, and returns the MappingRuleResult, if
+ * the rule matches, skip action otherwise.
+ * @param variables The variable context, which contains all the variables
+ * @return The rule's result or skip action if the rule doesn't apply
+ */
+ public MappingRuleResult evaluate(VariableContext variables) {
+ if (matcher.match(variables)) {
+ return action.execute(variables);
+ }
+
+ return MappingRuleResult.createSkipResult();
+ }
+
+ /**
+ * Returns the associated action's fallback.
+ * @return The fallback of the action
+ */
+ public MappingRuleResult getFallback() {
+ return action.getFallback();
+ }
+
+ /**
+ * Creates a MappingRule object from the legacy style configuration. The
+ * configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user).
+ * Using the source and path parts of the legacy rule, this method will
+ * create an application MappingRule which behaves as the legacy rule defined.
+ * This version of the method does not require type, since legacy application
+ * mappings omitted the 'a', so this method is to be used for those rules,
+ * which in all case are application mappings.
+ * @param source This part of the rule determines which applications the rule
+ * will be applied
+ * @param path The path where the application is to be placed
+ * @return MappingRule based on the provided settings
+ */
+ public static MappingRule createLegacyRule(String source, String path) {
+ return createLegacyRule(APPLICATION_MAPPING, source, path);
+ }
+
+ /**
+ * Creates a MappingRule object from the legacy style configuration. The
+ * configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user).
+ * Using the type, source and path parts of the legacy rule, this method will
+ * create a MappingRule which behaves as the legacy rule defined.
+ * @param type The type of the rule, can be
+ * 'u' for user mapping, 'g' for group mapping or
+ * 'a' for application mapping
+ * @param source This part of the rule determines which submissions this rule
+ * should apply to (eg. if type is 'u', source will match
+ * against the user name)
+ * @param path The path where the application is to be placed
+ * @return MappingRule based on the provided settings
+ */
+ public static MappingRule createLegacyRule(
+ String type, String source, String path) {
+ MappingRuleMatcher matcher;
+ MappingRuleAction action = new MappingRuleActions.PlaceToQueueAction(path);
+ //While legacy rule fallback handling is a bit inconsistent, the most cases
+ //it fall back to default queue placement, so this is the best approximation
+ action.setFallbackDefaultPlacement();
+
+ switch (type) {
+ case USER_MAPPING:
+ matcher = MappingRuleMatchers.createUserMatcher(source);
+ break;
+ case GROUP_MAPPING:
+ matcher = MappingRuleMatchers.createGroupMatcher(source);
+ break;
+ case APPLICATION_MAPPING:
+ matcher = MappingRuleMatchers.createApplicationNameMatcher(source);
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid mapping rule type '" +
+ type + "'");
+ }
+
+ return new MappingRule(matcher, action);
+ }
+
+ public void validate(MappingRuleValidationContext ctx)
+ throws YarnException {
+ this.action.validate(ctx);
+ }
+
+ @Override
+ public String toString() {
+ return "MappingRule{" +
+ "matcher=" + matcher +
+ ", action=" + action +
+ '}';
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java
new file mode 100644
index 0000000..2fa3140
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.util.StringUtils;
+import org.junit.Test;
+
+public class TestMappingRule {
+ VariableContext setupVariables(
+ String user, String group, String secGroup, String appName) {
+ VariableContext variables = new VariableContext();
+ variables.put("%default", "root.default");
+ variables.put("%user", user);
+ variables.put("%primary_group", group);
+ variables.put("%secondary_group", secGroup);
+ variables.put("%application", appName);
+ variables.put("%sub", "xxx");
+ variables.put("%empty", "");
+ variables.put("%null", null);
+ variables.setImmutables("%user", "%primary_group", "%secondary_group",
+ "%application");
+
+ return variables;
+ }
+
+ void assertSkipResult(MappingRuleResult result) {
+ assertTrue(
+ MappingRuleResultType.SKIP == result.getResult());
+ }
+
+ void assertPlaceResult(MappingRuleResult result, String queue) {
+ assertTrue(
+ MappingRuleResultType.PLACE == result.getResult());
+ assertEquals(queue, result.getQueue());
+ }
+
+ @Test
+ public void testMappingRuleEvaluation() {
+ VariableContext matching = setupVariables(
+ "bob", "developer", "users", "MR");
+ VariableContext mismatching = setupVariables(
+ "joe", "tester", "admins", "Spark");
+
+ MappingRule rule = new MappingRule(
+ MappingRuleMatchers.createUserMatcher("bob"),
+ (new MappingRuleActions.PlaceToQueueAction("%default.%default"))
+ .setFallbackSkip()
+ );
+
+ assertSkipResult(rule.getFallback());
+
+ MappingRuleResult matchingResult = rule.evaluate(matching);
+ MappingRuleResult mismatchingResult = rule.evaluate(mismatching);
+
+ assertSkipResult(mismatchingResult);
+ assertPlaceResult(matchingResult, "root.default.root.default");
+ }
+
+ MappingRule createMappingRuleFromLegacyString(String legacyMapping) {
+ String[] mapping =
+ StringUtils
+ .getTrimmedStringCollection(legacyMapping, ":")
+ .toArray(new String[] {});
+
+ if (mapping.length == 2) {
+ return MappingRule.createLegacyRule(mapping[0], mapping[1]);
+ }
+
+ return MappingRule.createLegacyRule(mapping[0], mapping[1], mapping[2]);
+ }
+
+ void evaluateLegacyStringTestcase(
+ String legacyString, VariableContext variables, String expectedQueue) {
+ MappingRule rule = createMappingRuleFromLegacyString(legacyString);
+ MappingRuleResult result = rule.evaluate(variables);
+ assertEquals(
+ rule.getFallback().getResult(), MappingRuleResultType.PLACE_TO_DEFAULT);
+
+ if (expectedQueue == null) {
+ assertSkipResult(result);
+ return;
+ }
+
+ assertPlaceResult(result, expectedQueue);
+ }
+
+ @Test
+ public void testLegacyEvaluation() {
+ VariableContext matching = setupVariables(
+ "bob", "developer", "users", "MR");
+ VariableContext mismatching = setupVariables(
+ "joe", "tester", "admins", "Spark");
+
+ evaluateLegacyStringTestcase(
+ "u:bob:root.%primary_group", matching, "root.developer");
+ evaluateLegacyStringTestcase(
+ "u:bob:root.%primary_group", mismatching, null);
+ evaluateLegacyStringTestcase(
+ "g:developer:%secondary_group.%user", matching, "users.bob");
+ evaluateLegacyStringTestcase(
+ "g:developer:%secondary_group.%user", mismatching, null);
+ evaluateLegacyStringTestcase(
+ "MR:root.static", matching, "root.static");
+ evaluateLegacyStringTestcase(
+ "MR:root.static", mismatching, null);
+
+ //catch all tests
+ evaluateLegacyStringTestcase(
+ "u:%user:root.%primary_group", matching, "root.developer");
+ evaluateLegacyStringTestcase(
+ "u:%user:root.%primary_group", mismatching, "root.tester");
+ }
+
+ @Test
+ public void testToStrings() {
+ MappingRuleAction action = new MappingRuleActions.PlaceToQueueAction(
+ "queue");
+ MappingRuleMatcher matcher = MappingRuleMatchers.createUserMatcher("bob");
+ MappingRule rule = new MappingRule(matcher, action);
+
+ assertEquals("MappingRule{matcher=" + matcher.toString() +
+ ", action=" + action.toString() + "}", rule.toString());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org