You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by kk...@apache.org on 2018/02/26 20:18:11 UTC
[2/2] hadoop git commit: YARN-7921. Transform a PlacementConstraint
to a string expression. Contributed by Weiwei Yang.
YARN-7921. Transform a PlacementConstraint to a string expression. Contributed by Weiwei Yang.
(cherry picked from commit e85188101c6c74b348a2fb6aa0f4e85c81b4a28c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb8765bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb8765bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb8765bb
Branch: refs/heads/branch-3.1
Commit: eb8765bbe9ddbf435247cc171113646c88c228c0
Parents: 33f8232
Author: Konstantinos Karanasos <kk...@apache.org>
Authored: Mon Feb 26 12:15:16 2018 -0800
Committer: Konstantinos Karanasos <kk...@apache.org>
Committed: Mon Feb 26 12:17:08 2018 -0800
----------------------------------------------------------------------
.../yarn/api/resource/PlacementConstraint.java | 141 ++++++++++++++++++-
.../resource/TestPlacementConstraintParser.java | 102 ++++++++++++--
.../TestPlacementConstraintTransformations.java | 7 +
3 files changed, 235 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8765bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index c054cbc..9bb17f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -23,6 +23,8 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -45,6 +47,11 @@ public class PlacementConstraint {
this.constraintExpr = constraintExpr;
}
+ @Override
+ public String toString() {
+ return this.constraintExpr.toString();
+ }
+
/**
* Get the constraint expression of the placement constraint.
*
@@ -226,6 +233,42 @@ public class PlacementConstraint {
}
@Override
+ public String toString() {
+ int max = getMaxCardinality();
+ int min = getMinCardinality();
+ List<String> targetExprList = getTargetExpressions().stream()
+ .map(TargetExpression::toString).collect(Collectors.toList());
+ List<String> targetConstraints = new ArrayList<>();
+ for (String targetExpr : targetExprList) {
+ if (min == 0 && max == 0) {
+ // anti-affinity
+ targetConstraints.add(new StringBuilder()
+ .append("notin").append(",")
+ .append(getScope()).append(",")
+ .append(targetExpr)
+ .toString());
+ } else if (min == 1 && max == Integer.MAX_VALUE) {
+ // affinity
+ targetConstraints.add(new StringBuilder()
+ .append("in").append(",")
+ .append(getScope()).append(",")
+ .append(targetExpr)
+ .toString());
+ } else {
+ // cardinality
+ targetConstraints.add(new StringBuilder()
+ .append("cardinality").append(",")
+ .append(getScope()).append(",")
+ .append(targetExpr).append(",")
+ .append(min).append(",")
+ .append(max)
+ .toString());
+ }
+ }
+ return String.join(":", targetConstraints);
+ }
+
+ @Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@@ -326,6 +369,23 @@ public class PlacementConstraint {
}
@Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ if (TargetType.ALLOCATION_TAG == this.targetType) {
+ // following by a comma separated tags
+ sb.append(String.join(",", getTargetValues()));
+ } else if (TargetType.NODE_ATTRIBUTE == this.targetType) {
+ // following by a comma separated key value pairs
+ if (this.getTargetValues() != null) {
+ String attributeName = this.getTargetKey();
+ String attributeValues = String.join(":", this.getTargetValues());
+ sb.append(attributeName + "=[" + attributeValues + "]");
+ }
+ }
+ return sb.toString();
+ }
+
+ @Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@@ -345,7 +405,16 @@ public class PlacementConstraint {
* TargetOperator enum helps to specify type.
*/
enum TargetOperator {
- IN, NOT_IN
+ IN("in"), NOT_IN("notin");
+
+ private String operator;
+ TargetOperator(String op) {
+ this.operator = op;
+ }
+
+ String getOperator() {
+ return this.operator;
+ }
}
private TargetOperator op;
@@ -415,6 +484,17 @@ public class PlacementConstraint {
}
@Override
+ public String toString() {
+ List<String> targetExprs = getTargetExpressions().stream().map(
+ targetExpression -> new StringBuilder()
+ .append(op.getOperator()).append(",")
+ .append(scope).append(",")
+ .append(targetExpression.toString())
+ .toString()).collect(Collectors.toList());
+ return String.join(":", targetExprs);
+ }
+
+ @Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@@ -517,6 +597,17 @@ public class PlacementConstraint {
+ (allocationTags != null ? allocationTags.hashCode() : 0);
return result;
}
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("cardinality").append(",").append(getScope()).append(",");
+ for (String tag : getAllocationTags()) {
+ sb.append(tag).append(",");
+ }
+ sb.append(minCardinality).append(",").append(maxCardinality);
+ return sb.toString();
+ }
}
/**
@@ -580,6 +671,22 @@ public class PlacementConstraint {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("and(");
+ Iterator<AbstractConstraint> it = getChildren().iterator();
+ while (it.hasNext()) {
+ AbstractConstraint child = it.next();
+ sb.append(child.toString());
+ if (it.hasNext()) {
+ sb.append(":");
+ }
+ }
+ sb.append(")");
+ return sb.toString();
+ }
}
/**
@@ -606,6 +713,22 @@ public class PlacementConstraint {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("or(");
+ Iterator<AbstractConstraint> it = getChildren().iterator();
+ while (it.hasNext()) {
+ AbstractConstraint child = it.next();
+ sb.append(child.toString());
+ if (it.hasNext()) {
+ sb.append(":");
+ }
+ }
+ sb.append(")");
+ return sb.toString();
+ }
}
/**
@@ -636,6 +759,22 @@ public class PlacementConstraint {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append("DelayedOr(");
+ Iterator<TimedPlacementConstraint> it = getChildren().iterator();
+ while (it.hasNext()) {
+ TimedPlacementConstraint child = it.next();
+ sb.append(child.toString());
+ if (it.hasNext()) {
+ sb.append(",");
+ }
+ }
+ sb.append(")");
+ return sb.toString();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8765bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
index 941f971..a69571c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintParser.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
import org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException;
@@ -51,42 +52,50 @@ public class TestPlacementConstraintParser {
@Test
public void testTargetExpressionParser()
throws PlacementConstraintParseException {
+ String expressionStr;
ConstraintParser parser;
AbstractConstraint constraint;
SingleConstraint single;
// Anti-affinity with single target tag
- // NOTIN,NDOE,foo
- parser = new TargetConstraintParser("NOTIN, NODE, foo");
+ // NOTIN,NODE,foo
+ expressionStr = "NOTIN, NODE, foo";
+ parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
Assert.assertEquals("node", single.getScope());
Assert.assertEquals(0, single.getMinCardinality());
Assert.assertEquals(0, single.getMaxCardinality());
+ verifyConstraintToString(expressionStr, constraint);
// lower cases is also valid
- parser = new TargetConstraintParser("notin, node, foo");
+ expressionStr = "notin, node, foo";
+ parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
Assert.assertEquals("node", single.getScope());
Assert.assertEquals(0, single.getMinCardinality());
Assert.assertEquals(0, single.getMaxCardinality());
+ verifyConstraintToString(expressionStr, constraint);
// Affinity with single target tag
// IN,NODE,foo
- parser = new TargetConstraintParser("IN, NODE, foo");
+ expressionStr = "IN, NODE, foo";
+ parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
Assert.assertEquals("node", single.getScope());
Assert.assertEquals(1, single.getMinCardinality());
Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality());
+ verifyConstraintToString(expressionStr, constraint);
// Anti-affinity with multiple target tags
// NOTIN,NDOE,foo,bar,exp
- parser = new TargetConstraintParser("NOTIN, NODE, foo, bar, exp");
+ expressionStr = "NOTIN, NODE, foo, bar, exp";
+ parser = new TargetConstraintParser(expressionStr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
@@ -98,6 +107,7 @@ public class TestPlacementConstraintParser {
single.getTargetExpressions().iterator().next();
Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString());
Assert.assertEquals(3, exp.getTargetValues().size());
+ verifyConstraintToString(expressionStr, constraint);
// Invalid OP
parser = new TargetConstraintParser("XYZ, NODE, foo");
@@ -112,12 +122,14 @@ public class TestPlacementConstraintParser {
@Test
public void testCardinalityConstraintParser()
throws PlacementConstraintParseException {
+ String expressionExpr;
ConstraintParser parser;
AbstractConstraint constraint;
SingleConstraint single;
// cardinality,NODE,foo,0,1
- parser = new CardinalityConstraintParser("cardinality, NODE, foo, 0, 1");
+ expressionExpr = "cardinality, NODE, foo, 0, 1";
+ parser = new CardinalityConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
@@ -130,10 +142,11 @@ public class TestPlacementConstraintParser {
Assert.assertEquals("ALLOCATION_TAG", exp.getTargetType().toString());
Assert.assertEquals(1, exp.getTargetValues().size());
Assert.assertEquals("foo", exp.getTargetValues().iterator().next());
+ verifyConstraintToString(expressionExpr, constraint);
// cardinality,NODE,foo,bar,moo,0,1
- parser = new CardinalityConstraintParser(
- "cardinality,RACK,foo,bar,moo,0,1");
+ expressionExpr = "cardinality,RACK,foo,bar,moo,0,1";
+ parser = new CardinalityConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof SingleConstraint);
single = (SingleConstraint) constraint;
@@ -147,6 +160,7 @@ public class TestPlacementConstraintParser {
Set<String> expectedTags = Sets.newHashSet("foo", "bar", "moo");
Assert.assertTrue(Sets.difference(expectedTags, exp.getTargetValues())
.isEmpty());
+ verifyConstraintToString(expressionExpr, constraint);
// Invalid scope string
try {
@@ -174,25 +188,29 @@ public class TestPlacementConstraintParser {
@Test
public void testAndConstraintParser()
throws PlacementConstraintParseException {
+ String expressionExpr;
ConstraintParser parser;
AbstractConstraint constraint;
And and;
- parser = new ConjunctionConstraintParser(
- "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)");
+ expressionExpr = "AND(NOTIN,NODE,foo:NOTIN,NODE,bar)";
+ parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof And);
and = (And) constraint;
Assert.assertEquals(2, and.getChildren().size());
+ verifyConstraintToString(expressionExpr, constraint);
- parser = new ConjunctionConstraintParser(
- "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)");
+ expressionExpr = "AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)";
+ parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof And);
Assert.assertEquals(2, and.getChildren().size());
+ verifyConstraintToString(expressionExpr, constraint);
- parser = new ConjunctionConstraintParser(
- "AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))");
+ expressionExpr =
+ "AND(NOTIN,NODE,foo:AND(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))";
+ parser = new ConjunctionConstraintParser(expressionExpr);
constraint = parser.parse();
Assert.assertTrue(constraint instanceof And);
and = (And) constraint;
@@ -200,6 +218,43 @@ public class TestPlacementConstraintParser {
Assert.assertTrue(and.getChildren().get(1) instanceof And);
and = (And) and.getChildren().get(1);
Assert.assertEquals(2, and.getChildren().size());
+ verifyConstraintToString(expressionExpr, constraint);
+ }
+
+ @Test
+ public void testOrConstraintParser()
+ throws PlacementConstraintParseException {
+ String expressionExpr;
+ ConstraintParser parser;
+ AbstractConstraint constraint;
+ Or or;
+
+ expressionExpr = "OR(NOTIN,NODE,foo:NOTIN,NODE,bar)";
+ parser = new ConjunctionConstraintParser(expressionExpr);
+ constraint = parser.parse();
+ Assert.assertTrue(constraint instanceof Or);
+ or = (Or) constraint;
+ Assert.assertEquals(2, or.getChildren().size());
+ verifyConstraintToString(expressionExpr, constraint);
+
+ expressionExpr = "OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1)";
+ parser = new ConjunctionConstraintParser(expressionExpr);
+ constraint = parser.parse();
+ Assert.assertTrue(constraint instanceof Or);
+ Assert.assertEquals(2, or.getChildren().size());
+ verifyConstraintToString(expressionExpr, constraint);
+
+ expressionExpr =
+ "OR(NOTIN,NODE,foo:OR(NOTIN,NODE,foo:cardinality,NODE,foo,0,1))";
+ parser = new ConjunctionConstraintParser(expressionExpr);
+ constraint = parser.parse();
+ Assert.assertTrue(constraint instanceof Or);
+ or = (Or) constraint;
+ Assert.assertTrue(or.getChildren().get(0) instanceof SingleConstraint);
+ Assert.assertTrue(or.getChildren().get(1) instanceof Or);
+ or = (Or) or.getChildren().get(1);
+ Assert.assertEquals(2, or.getChildren().size());
+ verifyConstraintToString(expressionExpr, constraint);
}
@Test
@@ -369,4 +424,23 @@ public class TestPlacementConstraintParser {
expectedPc2 = targetNotIn("node", allocationTag("foo")).build();
Assert.assertEquals(expectedPc2, actualPc2);
}
+
+ // We verify the toString result by parsing it again
+ // instead of raw string comparing. This is because internally
+ // we are not storing tags strictly to its original order, so
+ // the toString result might have different ordering with the
+ // input expression.
+ private void verifyConstraintToString(String inputExpr,
+ AbstractConstraint constraint) {
+ String constrainExpr = constraint.toString();
+ System.out.println("Input: " + inputExpr
+ .toLowerCase().replaceAll(" ", ""));
+ System.out.println("ToString: " + constrainExpr);
+ try {
+ PlacementConstraintParser.parseExpression(constrainExpr);
+ } catch (PlacementConstraintParseException e) {
+ Assert.fail("The parser is unable to parse the expression: "
+ + constrainExpr + ", caused by: " + e.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8765bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
index aa92d7a..557b82f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
@@ -65,6 +65,10 @@ public class TestPlacementConstraintTransformations {
SingleConstraint single = (SingleConstraint) sConstraintExpr;
TargetConstraint target = (TargetConstraint) tConstraintExpr;
+
+ // Make sure the expression string is consistent
+ // before and after transforming
+ Assert.assertEquals(single.toString(), target.toString());
Assert.assertEquals(single.getScope(), target.getScope());
Assert.assertEquals(TargetOperator.IN, target.getOp());
Assert.assertEquals(single.getTargetExpressions(),
@@ -101,6 +105,9 @@ public class TestPlacementConstraintTransformations {
Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ // Make sure the consistent expression string is consistent
+ // before and after transforming
+ Assert.assertEquals(single.toString(), cardinality.toString());
Assert.assertEquals(cardinality.getScope(), single.getScope());
Assert.assertEquals(cardinality.getMinCardinality(),
single.getMinCardinality());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org