You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2016/05/24 18:11:35 UTC
svn commit: r1745385 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/
test/org/apache/pig/test/ test/org/apache/pig/tez/
Author: daijy
Date: Tue May 24 18:11:35 2016
New Revision: 1745385
URL: http://svn.apache.org/viewvc?rev=1745385&view=rev
Log:
PIG-4902: Fix UT failures on 0.16 branch: TestTezGraceParallelism, TestPigScriptParser
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java
pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
pig/trunk/test/org/apache/pig/tez/TestTezGraceParallelism.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1745385&r1=1745384&r2=1745385&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue May 24 18:11:35 2016
@@ -139,6 +139,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4902: Fix UT failures on 0.16 branch: TestTezGraceParallelism, TestPigScriptParser (daijy)
+
PIG-4909: PigStorage incompatible with commons-cli-1.3 (knoguchi)
PIG-4908: JythonFunction refers to Oozie launcher script absolute path (rohini)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java?rev=1745385&r1=1745384&r2=1745385&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/optimizer/TezOperDependencyParallelismEstimator.java Tue May 24 18:11:35 2016
@@ -203,7 +203,7 @@ public class TezOperDependencyParallelis
if (successor != null) {
// Map side combiner
TezEdgeDescriptor edge = tezOp.outEdges.get(successor.getOperatorKey());
- if (!edge.combinePlan.isEmpty()) {
+ if (!edge.combinePlan.isEmpty() || edge.needsDistinctCombiner()) {
if (successor.isDistinct()) {
factor = DEFAULT_DISTINCT_FACTOR;
} else {
Modified: pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java?rev=1745385&r1=1745384&r2=1745385&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java Tue May 24 18:11:35 2016
@@ -102,7 +102,7 @@ public class TestPigScriptParser {
@Test
public void testDefineUDF() throws Exception {
- PigServer ps = new PigServer(ExecType.LOCAL);
+ PigServer ps = new PigServer(Util.getLocalTestMode());
String inputData[] = {
"dshfdskfwww.xyz.com/sportsjoadfjdslpdshfdskfwww.xyz.com/sportsjoadfjdsl" ,
"kas;dka;sd" ,
Modified: pig/trunk/test/org/apache/pig/tez/TestTezGraceParallelism.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezGraceParallelism.java?rev=1745385&r1=1745384&r2=1745385&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezGraceParallelism.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezGraceParallelism.java Tue May 24 18:11:35 2016
@@ -117,15 +117,15 @@ public class TestTezGraceParallelism {
Util.createLogAppender("testDecreaseParallelism", writer, new Class[]{PigGraceShuffleVertexManager.class, ShuffleVertexManager.class});
try {
// DAG: 47 \
- // -> 49(join) -> 52(distinct) -> 61(group)
+ // -> 49(join) -> 52(distinct) -> 56(group)
// 48 /
// Parallelism at compile time:
// DAG: 47(1) \
- // -> 49(2) -> 52(20) -> 61(200)
+ // -> 49(2) -> 52(20) -> 56(200)
// 48(1) /
// However, when 49 finishes, the actual output of 49 only justify parallelism 1.
- // We adjust the parallelism for 61 to 100 based on this.
- // At runtime, ShuffleVertexManager still kick in and further reduce parallelism from 100 to 1.
+ // We adjust the parallelism for 56 to 7 based on this.
+ // At runtime, ShuffleVertexManager still kick in and further reduce parallelism from 7 to 1.
//
pigServer.registerQuery("A = load '" + INPUT_DIR + "/" + INPUT_FILE1 + "' as (name:chararray, age:int);");
pigServer.registerQuery("B = load '" + INPUT_DIR + "/" + INPUT_FILE2 + "' as (name:chararray, gender:chararray);");
@@ -140,10 +140,10 @@ public class TestTezGraceParallelism {
"('F',1349L)", "('M',1373L)"});
Util.checkQueryOutputsAfterSort(iter, expectedResults);
assertTrue(writer.toString().contains("Initialize parallelism for scope-52 to 18"));
- assertTrue(writer.toString().contains("Initialize parallelism for scope-61 to 7"));
+ assertTrue(writer.toString().contains("Initialize parallelism for scope-56 to 7"));
assertTrue(writer.toString().contains("Reduce auto parallelism for vertex: scope-49 to 1 from 2"));
assertTrue(writer.toString().contains("Reduce auto parallelism for vertex: scope-52 to 1 from 18"));
- assertTrue(writer.toString().contains("Reduce auto parallelism for vertex: scope-61 to 1 from 7"));
+ assertTrue(writer.toString().contains("Reduce auto parallelism for vertex: scope-56 to 1 from 7"));
} finally {
Util.removeLogAppender("testDecreaseParallelism", PigGraceShuffleVertexManager.class, ShuffleVertexManager.class);
}
@@ -217,8 +217,8 @@ public class TestTezGraceParallelism {
count++;
}
assertEquals(count, 20);
- assertTrue(writer.toString().contains("All predecessors for scope-84 are finished, time to set parallelism for scope-85"));
- assertTrue(writer.toString().contains("Initialize parallelism for scope-85 to 10"));
+ assertTrue(writer.toString().contains("All predecessors for scope-79 are finished, time to set parallelism for scope-80"));
+ assertTrue(writer.toString().contains("Initialize parallelism for scope-80 to 10"));
} finally {
Util.removeLogAppender("testJoinWithDifferentDepth", PigGraceShuffleVertexManager.class);
}
@@ -262,9 +262,9 @@ public class TestTezGraceParallelism {
StringWriter writer = new StringWriter();
Util.createLogAppender("testJoinWithUnion", writer, PigGraceShuffleVertexManager.class);
try {
- // DAG: 29 -> 32 -> 41 \
- // -> 70 (vertex group) -> 61
- // 42 -> 45 -> 54 /
+ // DAG: 29 -> 32 -> 36 \
+ // -> 55 (vertex group) -> 51
+ // 37 -> 40 -> 44 /
pigServer.registerQuery("A = load '" + INPUT_DIR + "/" + INPUT_FILE2 + "' as (name:chararray, gender:chararray);");
pigServer.registerQuery("B = distinct A;");
pigServer.registerQuery("C = group B by name;");
@@ -280,8 +280,8 @@ public class TestTezGraceParallelism {
count++;
}
assertEquals(count, 20);
- assertTrue(writer.toString().contains("time to set parallelism for scope-41"));
- assertTrue(writer.toString().contains("time to set parallelism for scope-54"));
+ assertTrue(writer.toString().contains("time to set parallelism for scope-36"));
+ assertTrue(writer.toString().contains("time to set parallelism for scope-44"));
} finally {
Util.removeLogAppender("testJoinWithUnion", PigGraceShuffleVertexManager.class);
}