You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2010/02/20 02:31:20 UTC
svn commit: r912057 - in /hadoop/pig/trunk: ./ src/org/apache/pig/builtin/
test/org/apache/pig/test/
Author: olga
Date: Sat Feb 20 01:31:20 2010
New Revision: 912057
URL: http://svn.apache.org/viewvc?rev=912057&view=rev
Log:
PIG-1233: NullPointerException in AVG (ankur via olgan)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/builtin/AVG.java
hadoop/pig/trunk/src/org/apache/pig/builtin/DoubleAvg.java
hadoop/pig/trunk/src/org/apache/pig/builtin/FloatAvg.java
hadoop/pig/trunk/src/org/apache/pig/builtin/IntAvg.java
hadoop/pig/trunk/src/org/apache/pig/builtin/LongAvg.java
hadoop/pig/trunk/test/org/apache/pig/test/TestAccumulator.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Sat Feb 20 01:31:20 2010
@@ -55,6 +55,8 @@
IMPROVEMENTS
+PIG-1233: NullPointerException in AVG (ankur via olgan)
+
PIG-1218: Use distributed cache to store samples (rding via pradeepkth)
PIG-1226: suuport for additional jar files (thejas via olgan)
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/AVG.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/AVG.java?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/AVG.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/AVG.java Sat Feb 20 01:31:20 2010
@@ -306,7 +306,7 @@
@Override
public Double getValue() {
Double avg = null;
- if (intermediateCount > 0) {
+ if (intermediateCount != null && intermediateCount > 0) {
avg = new Double(intermediateSum / intermediateCount);
}
return avg;
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/DoubleAvg.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/DoubleAvg.java?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/DoubleAvg.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/DoubleAvg.java Sat Feb 20 01:31:20 2010
@@ -277,7 +277,7 @@
@Override
public Double getValue() {
Double avg = null;
- if (intermediateCount > 0) {
+ if (intermediateCount != null && intermediateCount > 0) {
avg = new Double(intermediateSum / intermediateCount);
}
return avg;
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/FloatAvg.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/FloatAvg.java?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/FloatAvg.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/FloatAvg.java Sat Feb 20 01:31:20 2010
@@ -274,7 +274,7 @@
@Override
public Double getValue() {
Double avg = null;
- if (intermediateCount > 0) {
+ if (intermediateCount != null && intermediateCount > 0) {
avg = new Double(intermediateSum / intermediateCount);
}
return avg;
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/IntAvg.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/IntAvg.java?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/IntAvg.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/IntAvg.java Sat Feb 20 01:31:20 2010
@@ -279,7 +279,7 @@
@Override
public Double getValue() {
Double avg = null;
- if (intermediateCount > 0) {
+ if (intermediateCount != null && intermediateCount > 0) {
avg = new Double(intermediateSum / intermediateCount);
}
return avg;
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/LongAvg.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/LongAvg.java?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/LongAvg.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/LongAvg.java Sat Feb 20 01:31:20 2010
@@ -274,7 +274,7 @@
@Override
public Double getValue() {
Double avg = null;
- if (intermediateCount > 0) {
+ if (intermediateCount != null && intermediateCount > 0) {
avg = new Double(intermediateSum / intermediateCount);
}
return avg;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestAccumulator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestAccumulator.java?rev=912057&r1=912056&r2=912057&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestAccumulator.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestAccumulator.java Sat Feb 20 01:31:20 2010
@@ -43,6 +43,9 @@
pigServer.getPigContext().getProperties().setProperty("pig.accumulative.batchsize", "2");
pigServer.getPigContext().getProperties().setProperty("pig.exec.batchsize", "2");
pigServer.getPigContext().getProperties().setProperty("pig.exec.nocombiner", "true");
+ // reducing the number of retry attempts to speed up test completion
+ pigServer.getPigContext().getProperties().setProperty("mapred.map.max.attempts","1");
+ pigServer.getPigContext().getProperties().setProperty("mapred.reduce.max.attempts","1");
}
@Before
@@ -86,6 +89,8 @@
w.println("200\t3.1");
w.println("100\t5.0");
w.println("300\t3.3");
+ w.println("400\t");
+ w.println("400\t");
w.close();
Util.copyFromLocalToCluster(cluster, INPUT_FILE3, INPUT_FILE3);
@@ -407,15 +412,49 @@
}
}
+ public void testAccumWithBuildinAvg() throws IOException {
+ HashMap<Integer, Double> expected = new HashMap<Integer, Double>();
+ expected.put(100, 3.0);
+ expected.put(200, 2.1);
+ expected.put(300, 3.3);
+ expected.put(400, null);
+ // Test all the averages for correct behaviour with null values
+ String[] types = { "double", "float", "int", "long" };
+ for (int i = 0; i < types.length; i++) {
+ if (i > 1) { // adjust decimal error for non real types
+ expected.put(200, 2.0);
+ expected.put(300, 3.0);
+ }
+ pigServer.registerQuery("A = load '" + INPUT_FILE3 + "' as (id:int, v:"
+ + types[i] + ");");
+ pigServer.registerQuery("C = group A by id;");
+ pigServer.registerQuery("D = foreach C generate group, AVG(A.v);");
+ Iterator<Tuple> iter = pigServer.openIterator("D");
+
+ while (iter.hasNext()) {
+ Tuple t = iter.next();
+ Double v = expected.get((Integer) t.get(0));
+ if (v != null) {
+ assertEquals(v.doubleValue(), ((Number) t.get(1)).doubleValue(),
+ 0.0001);
+ } else {
+ assertEquals(null, t.get(1));
+ }
+ }
+ }
+ }
+
public void testAccumWithBuildin() throws IOException{
pigServer.registerQuery("A = load '" + INPUT_FILE3 + "' as (id:int, v:double);");
pigServer.registerQuery("C = group A by id;");
- pigServer.registerQuery("D = foreach C generate group, SUM(A.v), AVG(A.v), COUNT(A.v), MIN(A.v), MAX(A.v);");
+ // moving AVG accumulator test to separate test case
+ pigServer.registerQuery("D = foreach C generate group, SUM(A.v), COUNT(A.v), MIN(A.v), MAX(A.v);");
HashMap<Integer, Double[]> expected = new HashMap<Integer, Double[]>();
- expected.put(100, new Double[]{15.0,3.0,5.0,1.0,5.0});
- expected.put(200, new Double[]{6.3,2.1,3.0,1.1,3.1});
- expected.put(300, new Double[]{3.3,3.3,1.0,3.3,3.3});
+ expected.put(100, new Double[]{15.0, 5.0, 1.0, 5.0});
+ expected.put(200, new Double[]{6.3, 3.0, 1.1, 3.1});
+ expected.put(300, new Double[]{3.3, 1.0, 3.3, 3.3});
+ expected.put(400, new Double[] { null, 0.0, null, null });
Iterator<Tuple> iter = pigServer.openIterator("D");
@@ -423,7 +462,12 @@
Tuple t = iter.next();
Double[] v = expected.get((Integer)t.get(0));
for(int i=0; i<v.length; i++) {
- assertEquals(v[i].doubleValue(), ((Number)t.get(i+1)).doubleValue(), 0.0001);
+ if (v[i] != null) {
+ assertEquals(v[i].doubleValue(), ((Number) t.get(i + 1))
+ .doubleValue(), 0.0001);
+ } else {
+ assertEquals(null, t.get(i + 1));
+ }
}
}
}