You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2015/05/08 00:59:51 UTC
svn commit: r1678272 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/
Author: daijy
Date: Thu May 7 22:59:50 2015
New Revision: 1678272
URL: http://svn.apache.org/r1678272
Log:
PIG-4537: Fix unit test failure introduced by TEZ-2392: TestCollectedGroup, TestLimitVariable, TestMapSideCogroup, etc
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POCounterStatsTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POIdentityInOutTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/PORankTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POSimpleTezLoad.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1678272&r1=1678271&r2=1678272&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 7 22:59:50 2015
@@ -76,6 +76,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4537: Fix unit test failure introduced by TEZ-2392: TestCollectedGroup, TestLimitVariable, TestMapSideCogroup, etc (daijy)
+
PIG-4530: StackOverflow in TestMultiQueryLocal running under hadoop20 (nielsbasjes via rohini)
PIG-4529: Pig on tez hit counter limit imposed by MR (daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POCounterStatsTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POCounterStatsTez.java?rev=1678272&r1=1678271&r2=1678272&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POCounterStatsTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POCounterStatsTez.java Thu May 7 22:59:50 2015
@@ -56,6 +56,7 @@ public class POCounterStatsTez extends P
// KeyValuesReader. After TEZ-661, switch to unsorted shuffle
private transient KeyValuesReader reader;
private transient KeyValueWriter writer;
+ private transient boolean finished = false;
public POCounterStatsTez(OperatorKey k) {
super(k);
@@ -123,6 +124,9 @@ public class POCounterStatsTez extends P
@Override
public Result getNextTuple() throws ExecException {
try {
+ if (finished) {
+ return RESULT_EOP;
+ }
Map<Integer, Long> counterRecords = new HashMap<Integer, Long>();
Integer key = null;
Long value = null;
@@ -151,6 +155,7 @@ public class POCounterStatsTez extends P
Tuple tuple = TupleFactory.getInstance().newTuple(1);
tuple.set(0, counterOffsets);
writer.write(POValueOutputTez.EMPTY_KEY, tuple);
+ finished = true;
return RESULT_EOP;
} catch (IOException e) {
throw new ExecException(e);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POIdentityInOutTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POIdentityInOutTez.java?rev=1678272&r1=1678271&r2=1678272&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POIdentityInOutTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POIdentityInOutTez.java Thu May 7 22:59:50 2015
@@ -56,6 +56,7 @@ public class POIdentityInOutTez extends
private transient KeyValueReader reader;
private transient KeyValuesReader shuffleReader;
private transient boolean shuffleInput;
+ private transient boolean finished = false;
public POIdentityInOutTez(OperatorKey k, POLocalRearrange inputRearrange) {
super(inputRearrange);
@@ -121,6 +122,9 @@ public class POIdentityInOutTez extends
@Override
public Result getNextTuple() throws ExecException {
try {
+ if (finished) {
+ return RESULT_EOP;
+ }
if (shuffleInput) {
while (shuffleReader.next()) {
Object curKey = shuffleReader.getCurrentKey();
@@ -152,6 +156,7 @@ public class POIdentityInOutTez extends
}
}
}
+ finished = true;
return RESULT_EOP;
} catch (IOException e) {
throw new ExecException(e);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/PORankTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/PORankTez.java?rev=1678272&r1=1678271&r2=1678272&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/PORankTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/PORankTez.java Thu May 7 22:59:50 2015
@@ -50,6 +50,7 @@ public class PORankTez extends PORank im
private transient KeyValueReader reader;
private transient Map<Integer, Long> counterOffsets;
private transient Configuration conf;
+ private transient boolean finished = false;
public PORankTez(PORank copy) {
super(copy);
@@ -133,6 +134,9 @@ public class PORankTez extends PORank im
@Override
public Result getNextTuple() throws ExecException {
+ if (finished) {
+ return RESULT_EOP;
+ }
Result inp = null;
try {
@@ -150,6 +154,7 @@ public class PORankTez extends PORank im
if (Boolean.valueOf(conf.get(JobControlCompiler.END_OF_INP_IN_MAP, "false"))) {
this.parentPlan.endOfAllInput = true;
}
+ finished = true;
return RESULT_EOP;
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POSimpleTezLoad.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POSimpleTezLoad.java?rev=1678272&r1=1678271&r2=1678272&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POSimpleTezLoad.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/operator/POSimpleTezLoad.java Thu May 7 22:59:50 2015
@@ -50,6 +50,7 @@ public class POSimpleTezLoad extends POL
private MRInput input;
private KeyValueReader reader;
private transient Configuration conf;
+ private transient boolean finished = false;
public POSimpleTezLoad(OperatorKey k, FileSpec lfile) {
super(k, lfile);
@@ -102,6 +103,9 @@ public class POSimpleTezLoad extends POL
@Override
public Result getNextTuple() throws ExecException {
try {
+ if (finished) {
+ return RESULT_EOP;
+ }
Result res = new Result();
if (!reader.next()) {
res.result = null;
@@ -112,6 +116,7 @@ public class POSimpleTezLoad extends POL
if (Boolean.valueOf(conf.get(JobControlCompiler.END_OF_INP_IN_MAP, "false"))) {
this.parentPlan.endOfAllInput = true;
}
+ finished = true;
} else {
Tuple next = (Tuple) reader.getCurrentValue();
res.result = next;