You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/12/02 20:47:38 UTC
svn commit: r1209658 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/pigstats/JobStats.java
test/org/apache/pig/test/TestPigRunner.java
Author: daijy
Date: Fri Dec 2 19:47:37 2011
New Revision: 1209658
URL: http://svn.apache.org/viewvc?rev=1209658&view=rev
Log:
PIG-2358: JobStats.getHadoopCounters() is never set and always returns null
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1209658&r1=1209657&r2=1209658&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Dec 2 19:47:37 2011
@@ -190,6 +190,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2358: JobStats.getHadoopCounters() is never set and always returns null (xutingz via daijy)
+
PIG-2184: Not able to provide positional reference to macro invocations (xutingz via daijy)
PIG-2209: JsonMetadata fails to find schema for glob paths (daijy)
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java?rev=1209658&r1=1209657&r2=1209658&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java Fri Dec 2 19:47:37 2011
@@ -332,7 +332,6 @@ public final class JobStats extends Oper
@SuppressWarnings("deprecation")
void addCounters(RunningJob rjob) {
- Counters counters = null;
if (rjob != null) {
try {
counters = rjob.getCounters();
Modified: pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=1209658&r1=1209657&r2=1209658&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigRunner.java Fri Dec 2 19:47:37 2011
@@ -35,6 +35,7 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.Counters;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigRunner.ReturnCode;
@@ -772,6 +773,44 @@ public class TestPigRunner {
}
}
+ @Test //Pig-2358
+ public void testGetHadoopCounters() throws Exception {
+ final String OUTPUT_FILE_2 = "output2";
+
+ PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
+ w.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+ w.println("B = filter A by a0 >= 4;");
+ w.println("C = filter A by a0 < 4;");
+ w.println("D = group C by a0;");
+ w.println("E = foreach D generate group, COUNT(C);");
+ w.println("store B into '" + OUTPUT_FILE_2 + "';");
+ w.println("store E into '" + OUTPUT_FILE + "';");
+ w.close();
+
+ try {
+ String[] args = { PIG_FILE };
+ PigStats stats = PigRunner.run(args, new TestNotificationListener());
+
+ Counters counter= ((JobStats)stats.getJobGraph().getSinks().get(0)).getHadoopCounters();
+ assertEquals(5, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+ PigStatsUtil.MAP_INPUT_RECORDS).getValue());
+ assertEquals(3, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+ PigStatsUtil.MAP_OUTPUT_RECORDS).getValue());
+ assertEquals(2, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+ PigStatsUtil.REDUCE_INPUT_RECORDS).getValue());
+ assertEquals(0, counter.getGroup(PigStatsUtil.TASK_COUNTER_GROUP).getCounterForName(
+ PigStatsUtil.REDUCE_OUTPUT_RECORDS).getValue());
+ assertEquals(20,counter.getGroup(PigStatsUtil.FS_COUNTER_GROUP).getCounterForName(
+ PigStatsUtil.HDFS_BYTES_WRITTEN).getValue());
+ assertEquals(30,counter.getGroup(PigStatsUtil.FS_COUNTER_GROUP).getCounterForName(
+ PigStatsUtil.HDFS_BYTES_READ).getValue());
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ Util.deleteFile(cluster, OUTPUT_FILE_2);
+ }
+ }
+
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {