You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/01/31 19:25:47 UTC
svn commit: r1563200 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
test/org/apache/pig/test/TestRank3.java
Author: daijy
Date: Fri Jan 31 18:25:46 2014
New Revision: 1563200
URL: http://svn.apache.org/r1563200
Log:
PIG-3726: Ranking empty records leads to NullPointerException
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/test/org/apache/pig/test/TestRank3.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1563200&r1=1563199&r2=1563200&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Jan 31 18:25:46 2014
@@ -76,6 +76,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3726: Ranking empty records leads to NullPointerException (jarcec via daijy)
+
PIG-3652: Pigmix parser (PigPerformanceLoader) deletes chars during parsing (keren3000 via daijy)
PIG-3722: Udf deserialization for registered classes fails in local_mode (aniket486)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1563200&r1=1563199&r2=1563200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Jan 31 18:25:46 2014
@@ -110,6 +110,7 @@ import org.apache.pig.impl.util.ObjectSe
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil;
import org.apache.pig.tools.pigstats.mapreduce.MRScriptState;
/**
@@ -375,7 +376,24 @@ public class JobControlCompiler{
try {
counters = HadoopShims.getCounters(job);
- groupCounters = counters.getGroup(getGroupName(counters.getGroupNames()));
+
+ String groupName = getGroupName(counters.getGroupNames());
+ // In case that the counter group was not find, we need to find
+ // out why. Only acceptable state is that the relation has been
+ // empty.
+ if (groupName == null) {
+ Counter outputRecords =
+ counters.getGroup(MRPigStatsUtil.TASK_COUNTER_GROUP)
+ .getCounterForName(MRPigStatsUtil.MAP_OUTPUT_RECORDS);
+
+ if(outputRecords.getCounter() == 0) {
+ globalCounters.put(operationID, new ArrayList<Pair<String, Long>>());
+ return;
+ } else {
+ throw new RuntimeException("Did not found RANK counter group for operationId: " + operationID);
+ }
+ }
+ groupCounters = counters.getGroup(groupName);
Iterator<Counter> it = groupCounters.iterator();
HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();
Modified: pig/trunk/test/org/apache/pig/test/TestRank3.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRank3.java?rev=1563200&r1=1563199&r2=1563200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRank3.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRank3.java Fri Jan 31 18:25:46 2014
@@ -52,6 +52,7 @@ public class TestRank3 {
pigServer = new PigServer("local");
data = resetData(pigServer);
+ data.set("empty");
data.set(
"testcascade",
tuple(3,2,3),
@@ -142,6 +143,19 @@ public class TestRank3 {
verifyExpected(data.get("result"), expected);
}
+ // See PIG-3726
+ @Test
+ public void testRankEmptyRelation() throws Exception {
+ String query = "DATA = LOAD 'empty' USING mock.Storage();"
+ + "A = rank DATA;"
+ + "store A into 'empty_result' using mock.Storage();";
+
+ Util.registerMultiLineQuery(pigServer, query);
+
+ Set<Tuple> expected = ImmutableSet.of();
+ verifyExpected(data.get("empty_result"), expected);
+ }
+
public void verifyExpected(List<Tuple> out, Set<Tuple> expected) {
for (Tuple tup : out) {
assertTrue(expected + " contains " + tup, expected.contains(tup));