You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/04/02 01:45:37 UTC
svn commit: r1087934 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
src/org/apache/pig/tools/pigstats/ test/org/apache/pig/test/
Author: rding
Date: Fri Apr 1 23:45:36 2011
New Revision: 1087934
URL: http://svn.apache.org/viewvc?rev=1087934&view=rev
Log:
PIG-1984: Worng stats shown when there are multiple stores but same file names
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java
pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1087934&r1=1087933&r2=1087934&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr 1 23:45:36 2011
@@ -128,6 +128,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1984: Worng stats shown when there are multiple stores but same file names (rding)
+
PIG-1893: Pig report input size -1 for empty input file (rding)
PIG-1868: New logical plan fails when I have complex data types from udf
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1087934&r1=1087933&r2=1087934&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Apr 1 23:45:36 2011
@@ -500,8 +500,10 @@ public class JobControlCompiler{
nwJob.setOutputFormatClass(PigOutputFormat.class);
+ int idx = 0;
for (POStore sto: storeLocations) {
sto.setMultiStore(true);
+ sto.setIndex(idx++);
}
conf.set("pig.streaming.log.dir",
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java?rev=1087934&r1=1087933&r2=1087934&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POStore.java Fri Apr 1 23:45:36 2011
@@ -66,6 +66,9 @@ public class POStore extends PhysicalOpe
// flag to distinguish single store from multiquery store.
private boolean isMultiStore;
+ // the index of multiquery store to track counters
+ private int index;
+
// If we know how to reload the store, here's how. The lFile
// FileSpec is set in PigServer.postProcess. It can be used to
// reload this store, if the optimizer has the need.
@@ -271,4 +274,12 @@ public class POStore extends PhysicalOpe
}
return (Tuple) out;
}
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public int getIndex() {
+ return index;
+ }
}
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java?rev=1087934&r1=1087933&r2=1087934&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java Fri Apr 1 23:45:36 2011
@@ -110,7 +110,7 @@ public abstract class PigStatsUtil {
public static String getMultiStoreCounterName(POStore store) {
String shortName = getShortName(store.getSFile().getFileName());
return (shortName == null) ? null
- : MULTI_STORE_RECORD_COUNTER + shortName;
+ : MULTI_STORE_RECORD_COUNTER + "_" + store.getIndex() + "_" + shortName;
}
/**
Modified: pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=1087934&r1=1087933&r2=1087934&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigRunner.java Fri Apr 1 23:45:36 2011
@@ -510,6 +510,39 @@ public class TestPigRunner {
}
@Test
+ public void testDuplicateCounterName2() throws Exception {
+
+ PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
+ w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+ w1.println("B = filter A by a0 > 3;");
+ w1.println("store A into 'output';");
+ w1.println("store B into 'tmp/output';");
+ w1.close();
+
+ try {
+ String[] args = { PIG_FILE };
+ PigStats stats = PigRunner.run(args, new TestNotificationListener());
+
+ assertTrue(stats.isSuccessful());
+
+ assertEquals(1, stats.getNumberJobs());
+ List<OutputStats> outputs = stats.getOutputStats();
+ assertEquals(2, outputs.size());
+ for (OutputStats outstats : outputs) {
+ if (outstats.getLocation().endsWith("tmp/output")) {
+ assertEquals(2, outstats.getNumberRecords());
+ } else {
+ assertEquals(5, outstats.getNumberRecords());
+ }
+ }
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ Util.deleteFile(cluster, "tmp/output");
+ }
+ }
+
+ @Test
public void testRegisterExternalJar() throws Exception {
String[] args = { "-Dpig.additional.jars=pig-withouthadoop.jar",
"-Dmapred.job.queue.name=default",
@@ -613,7 +646,7 @@ public class TestPigRunner {
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
-
+
@Test //PIG-1893
public void testEmptyFileCounter() throws Exception {