You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/01/30 03:19:27 UTC
svn commit: r904710 - in /hadoop/pig/branches/branch-0.6: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/LOForEach.java
test/org/apache/pig/test/TestPruneColumn.java
Author: daijy
Date: Sat Jan 30 02:19:27 2010
New Revision: 904710
URL: http://svn.apache.org/viewvc?rev=904710&view=rev
Log:
PIG-1210: fieldsToRead send the same fields more than once in some cases
Modified:
hadoop/pig/branches/branch-0.6/CHANGES.txt
hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/CHANGES.txt?rev=904710&r1=904709&r2=904710&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.6/CHANGES.txt Sat Jan 30 02:19:27 2010
@@ -281,6 +281,8 @@
PIG-1195: POSort should take care of sort order (daijy)
+PIG-1210: fieldsToRead send the same fields more than once in some cases (daijy)
+
Release 0.5.0
INCOMPATIBLE CHANGES
Modified: hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=904710&r1=904709&r2=904710&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original)
+++ hadoop/pig/branches/branch-0.6/src/org/apache/pig/impl/logicalLayer/LOForEach.java Sat Jan 30 02:19:27 2010
@@ -866,7 +866,8 @@
ArrayList<Pair<Integer, Integer>> inputList = new ArrayList<Pair<Integer, Integer>>();
for (LOProject project : projectFinder.getProjectSet()) {
for (int inputColumn : project.getProjection()) {
- inputList.add(new Pair<Integer, Integer>(0, inputColumn));
+ if (!inputList.contains(new Pair<Integer, Integer>(0, inputColumn)))
+ inputList.add(new Pair<Integer, Integer>(0, inputColumn));
}
}
if (inputList.size()==0)
Modified: hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java?rev=904710&r1=904709&r2=904710&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/branches/branch-0.6/test/org/apache/pig/test/TestPruneColumn.java Sat Jan 30 02:19:27 2010
@@ -33,8 +33,12 @@
import org.apache.log4j.SimpleLayout;
import org.apache.pig.ExecType;
import org.apache.pig.FilterFunc;
+import org.apache.pig.LoadFunc;
import org.apache.pig.PigServer;
+import org.apache.pig.LoadFunc.RequiredFieldList;
+import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.optimizer.PruneColumns;
import org.junit.Before;
import org.junit.Test;
@@ -55,8 +59,24 @@
File tmpFile9;
File tmpFile10;
File logFile;
- Logger logger;
+ static public class PigStorageWithTrace extends PigStorage {
+
+ /**
+ * @param delimiter
+ */
+ public PigStorageWithTrace() {
+ super();
+ }
+ @Override
+ public LoadFunc.RequiredFieldResponse fieldsToRead(RequiredFieldList requiredFieldList) throws FrontendException {
+ LoadFunc.RequiredFieldResponse response = super.fieldsToRead(requiredFieldList);
+ Logger logger = Logger.getLogger(this.getClass());
+ logger.info(requiredFieldList);
+ return response;
+ }
+
+ }
private static final String simpleEchoStreamingCommand;
static {
if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
@@ -76,7 +96,7 @@
@Before
@Override
public void setUp() throws Exception{
- logger = Logger.getLogger(PruneColumns.class);
+ Logger logger = Logger.getLogger(PruneColumns.class);
logger.removeAllAppenders();
logger.setLevel(Level.INFO);
SimpleLayout layout = new SimpleLayout();
@@ -84,6 +104,10 @@
FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
logger.addAppender(appender);
+ Logger pigStorageWithTraceLogger = Logger.getLogger(PigStorageWithTrace.class);
+ pigStorageWithTraceLogger.setLevel(Level.INFO);
+ pigStorageWithTraceLogger.addAppender(appender);
+
pigServer = new PigServer("local");
//pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
tmpFile1 = File.createTempFile("prune", "txt");
@@ -1673,4 +1697,27 @@
assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0, $2",
"No map keys pruned for A", "No column pruned for B", "No map keys pruned for B"}));
}
+
+ // See PIG-1210
+ @Test
+ public void testFieldsToReadDuplicatedEntry() throws Exception {
+ pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' using "+PigStorageWithTrace.class.getName()
+ +" AS (a0, a1, a2);");
+ pigServer.registerQuery("B = foreach A generate a0+a0, a1, a2;");
+ Iterator<Tuple> iter = pigServer.openIterator("B");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(2.0,2,3)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(4.0,5,2)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
+ "No map keys pruned for A", "[0,1,2]"}));
+ }
+
}