You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2010/02/09 23:40:15 UTC
svn commit: r908262 [2/2] - in /hadoop/pig/branches/load-store-redesign: ./
src/docs/src/documentation/content/xdocs/
src/org/apache/pig/backend/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/execu...
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPruneColumn.java Tue Feb 9 22:40:13 2010
@@ -33,8 +33,11 @@
import org.apache.log4j.SimpleLayout;
import org.apache.pig.ExecType;
import org.apache.pig.FilterFunc;
+import org.apache.pig.LoadPushDown;
import org.apache.pig.PigServer;
+import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.optimizer.PruneColumns;
import org.junit.Before;
import org.junit.Test;
@@ -55,8 +58,24 @@
File tmpFile9;
File tmpFile10;
File logFile;
- Logger logger;
+ static public class PigStorageWithTrace extends PigStorage {
+
+ /**
+ * @param delimiter
+ */
+ public PigStorageWithTrace() {
+ super();
+ }
+ @Override
+ public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException {
+ RequiredFieldResponse response = super.pushProjection(requiredFieldList);
+ Logger logger = Logger.getLogger(this.getClass());
+ logger.info(requiredFieldList);
+ return response;
+ }
+
+ }
private static final String simpleEchoStreamingCommand;
static {
if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
@@ -76,7 +95,7 @@
@Before
@Override
public void setUp() throws Exception{
- logger = Logger.getLogger(PruneColumns.class);
+ Logger logger = Logger.getLogger(PruneColumns.class);
logger.removeAllAppenders();
logger.setLevel(Level.INFO);
SimpleLayout layout = new SimpleLayout();
@@ -84,6 +103,10 @@
FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
logger.addAppender(appender);
+ Logger pigStorageWithTraceLogger = Logger.getLogger(PigStorageWithTrace.class);
+ pigStorageWithTraceLogger.setLevel(Level.INFO);
+ pigStorageWithTraceLogger.addAppender(appender);
+
pigServer = new PigServer("local");
//pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
tmpFile1 = File.createTempFile("prune", "txt");
@@ -1767,4 +1790,59 @@
assertTrue(emptyLogFileMessage());
}
+
+ // See PIG-1184
+ @Test
+ public void testForEachFlatten() throws Exception {
+ File inputFile = Util.createInputFile("table_testForEachFlatten", "", new String[]{"oiue\tM\t{(3),(4)}\t{(toronto),(montreal)}"});
+
+ pigServer.registerQuery("A = load '"+inputFile.toString()+"' as (a0:chararray, a1:chararray, a2:bag{t:tuple(id:chararray)}, a3:bag{t:tuple(loc:chararray)});");
+ pigServer.registerQuery("B = foreach A generate a0, a1, flatten(a2), flatten(a3), 10;");
+ pigServer.registerQuery("C = foreach B generate a0, $4;");
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(oiue,10)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(oiue,10)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(oiue,10)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(oiue,10)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1",
+ "No map keys pruned for A"}));
+ }
+
+ // See PIG-1210
+ @Test
+ public void testFieldsToReadDuplicatedEntry() throws Exception {
+ pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' using "+PigStorageWithTrace.class.getName()
+ +" AS (a0, a1, a2);");
+ pigServer.registerQuery("B = foreach A generate a0+a0, a1, a2;");
+ Iterator<Tuple> iter = pigServer.openIterator("B");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(2.0,2,3)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(4.0,5,2)"));
+
+ assertFalse(iter.hasNext());
+
+ assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
+ "No map keys pruned for A", "[0,1,2]"}));
+ }
+
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSchema.java Tue Feb 9 22:40:13 2010
@@ -36,16 +36,22 @@
package org.apache.pig.test;
-import java.util.* ;
-
-import org.apache.pig.data.* ;
-import org.apache.pig.impl.logicalLayer.schema.* ;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import org.junit.* ;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
import junit.framework.Assert;
-import junit.framework.TestCase ;
+import junit.framework.TestCase;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.SchemaMergeException;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.junit.Test;
public class TestSchema extends TestCase {
@@ -634,4 +640,20 @@
Assert.assertTrue(Schema.FieldSchema.castable(castFieldSchema, inputFieldSchema));
}
}
+
+ public void testSchemaSerialization() throws IOException {
+ MiniCluster cluster = MiniCluster.buildCluster();
+ PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ String inputFileName = "testSchemaSerialization-input.txt";
+ String[] inputData = new String[] { "foo\t1", "hello\t2" };
+ Util.createInputFile(cluster, inputFileName, inputData);
+ String script = "a = load '"+ inputFileName +"' as (f1:chararray, f2:int);" +
+ " b = group a all; c = foreach b generate org.apache.pig.test.InputSchemaUDF(a);";
+ Util.registerMultiLineQuery(pigServer, script);
+ Iterator<Tuple> it = pigServer.openIterator("c");
+ while(it.hasNext()) {
+ Tuple t = it.next();
+ assertEquals("{a: {f1: chararray,f2: int}}", t.get(0));
+ }
+ }
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestStreamingLocal.java Tue Feb 9 22:40:13 2010
@@ -18,6 +18,8 @@
package org.apache.pig.test;
import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
import junit.framework.TestCase;
@@ -286,6 +288,39 @@
Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults);
}
}
+
+ @Test
+ public void testJoinTwoStreamingRelations()
+ throws Exception {
+ ArrayList<String> list = new ArrayList<String>();
+ for (int i=0; i<10000; i++) {
+ list.add("A," + i);
+ }
+ File input = Util.createInputFile("tmp", "", list.toArray(new String[0]));
+
+ // Expected results
+ Tuple expected = DefaultTupleFactory.getInstance().newTuple(4);
+ expected.set(0, "A");
+ expected.set(1, 0);
+ expected.set(2, "A");
+ expected.set(3, 0);
+
+ pigServer.registerQuery("A = load 'file:" + Util.encodeEscape(input.toString()) + "' using " +
+ PigStorage.class.getName() + "(',') as (a0, a1);");
+ pigServer.registerQuery("B = stream A through `head -1` as (a0, a1);");
+ pigServer.registerQuery("C = load 'file:" + Util.encodeEscape(input.toString()) + "' using " +
+ PigStorage.class.getName() + "(',') as (a0, a1);");
+ pigServer.registerQuery("D = stream C through `head -1` as (a0, a1);");
+ pigServer.registerQuery("E = join B by a0, D by a0;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("E");
+ int count = 0;
+ while (iter.hasNext()) {
+ Assert.assertEquals(expected.toString(), iter.next().toString());
+ count++;
+ }
+ Assert.assertTrue(count == 1);
+ }
@Test
public void testLocalNegativeLoadStoreOptimization() throws Exception {
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Tue Feb 9 22:40:13 2010
@@ -26,7 +26,7 @@
| |
| |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -152
|
- |---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -145:
+ |---MapReduce(20,SUM,COUNT,TestMRCompiler$WeirdComparator) - -145:
| Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -151
| |
| |---New For Each(false,false)[tuple] - -2197807331204639125
@@ -88,4 +88,4 @@
| | |
| | Constant(true) - --6860857982727545176
| |
- | |---Load(DummyFil:DummyLdr) - -737965411848544085
\ No newline at end of file
+ | |---Load(DummyFil:DummyLdr) - -737965411848544085
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java?rev=908262&r1=908261&r2=908262&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/GenPhyOp.java Tue Feb 9 22:40:13 2010
@@ -762,7 +762,6 @@
public static POStore topStoreOp() {
POStore ret = new POStore(new OperatorKey("", r.nextLong()));
- ret.setSFile(new FileSpec("DummyFil", new FuncSpec("DummyLdr")));
return ret;
}