You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/03/29 21:07:06 UTC
svn commit: r1086682 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/pigstats/JobStats.java
test/org/apache/pig/test/TestPigRunner.java
Author: rding
Date: Tue Mar 29 19:07:06 2011
New Revision: 1086682
URL: http://svn.apache.org/viewvc?rev=1086682&view=rev
Log:
PIG-1893: Pig report input size -1 for empty input file
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1086682&r1=1086681&r2=1086682&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 29 19:07:06 2011
@@ -124,6 +124,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1893: Pig report input size -1 for empty input file (rding)
+
PIG-1868: New logical plan fails when I have complex data types from udf
(daijy)
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java?rev=1086682&r1=1086681&r2=1086682&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java Tue Mar 29 19:07:06 2011
@@ -526,7 +526,7 @@ public final class JobStats extends Oper
}
private void addOneInputStats(String fileName, int index) {
- long records = -1;
+ long records = 0;
Long n = multiInputCounters.get(
PigStatsUtil.getMultiInputsCounterName(fileName, index));
if (n != null) {
Modified: pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=1086682&r1=1086681&r2=1086682&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigRunner.java Tue Mar 29 19:07:06 2011
@@ -614,6 +614,77 @@ public class TestPigRunner {
}
}
+
+ @Test //PIG-1893
+ public void testEmptyFileCounter() throws Exception {
+
+ PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
+ w.close();
+
+ Util.copyFromLocalToCluster(cluster, "myinputfile", "1.txt");
+
+ PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
+ w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+ w1.println("B = load '1.txt' as (a0:int, a1:int, a2:int);");
+ w1.println("C = join A by a0, B by a0;");
+ w1.println("store C into '" + OUTPUT_FILE + "';");
+ w1.close();
+
+ try {
+ String[] args = { PIG_FILE };
+ PigStats stats = PigRunner.run(args, new TestNotificationListener());
+
+ assertTrue(stats.isSuccessful());
+
+ assertEquals(1, stats.getNumberJobs());
+ List<InputStats> inputs = stats.getInputStats();
+ assertEquals(2, inputs.size());
+ for (InputStats instats : inputs) {
+ if (instats.getLocation().endsWith("1.txt")) {
+ assertEquals(0, instats.getNumberRecords());
+ } else {
+ assertEquals(5, instats.getNumberRecords());
+ }
+ }
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ }
+ }
+
+ @Test //PIG-1893
+ public void testEmptyFileCounter2() throws Exception {
+
+ PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
+ w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+ w1.println("B = filter A by a0 < 0;");
+ w1.println("store A into '" + OUTPUT_FILE + "';");
+ w1.println("store B into 'output2';");
+ w1.close();
+
+ try {
+ String[] args = { PIG_FILE };
+ PigStats stats = PigRunner.run(args, new TestNotificationListener());
+
+ assertTrue(stats.isSuccessful());
+
+ assertEquals(1, stats.getNumberJobs());
+ List<OutputStats> outputs = stats.getOutputStats();
+ assertEquals(2, outputs.size());
+ for (OutputStats outstats : outputs) {
+ if (outstats.getLocation().endsWith("output2")) {
+ assertEquals(0, outstats.getNumberRecords());
+ } else {
+ assertEquals(5, outstats.getNumberRecords());
+ }
+ }
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ Util.deleteFile(cluster, "output2");
+ }
+ }
+
public static class TestNotificationListener implements PigProgressNotificationListener {
private Map<String, int[]> numMap = new HashMap<String, int[]>();