You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by dv...@apache.org on 2012/10/16 00:14:47 UTC
svn commit: r1398536 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/te...
Author: dvryaboy
Date: Mon Oct 15 22:14:46 2012
New Revision: 1398536
URL: http://svn.apache.org/viewvc?rev=1398536&view=rev
Log:
PIG-2908: Fix unit tests to work with jdk7
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitionHelper.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
pig/trunk/src/org/apache/pig/ResourceSchema.java
pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java
pig/trunk/test/org/apache/pig/test/TestGFCross.java
pig/trunk/test/org/apache/pig/test/TestHBaseStorage.java
pig/trunk/test/org/apache/pig/test/TestJobControlCompiler.java
pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java
pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java
pig/trunk/test/org/apache/pig/test/TestMergeJoinOuter.java
pig/trunk/test/org/apache/pig/test/TestNewPlanLogToPhyTranslationVisitor.java
pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Oct 15 22:14:46 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2908: Fix unit tests to work with jdk7 (rohini via dvryaboy)
+
OPTIMIZATIONS
BUG FIXES
Modified: pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Mon Oct 15 22:14:46 2012
@@ -488,6 +488,7 @@
<ivy:cachepath pathid="mr-apps-test-ivy.classpath" conf="test" />
<path id="mr-apps-test.classpath">
+ <pathelement path="${build.classes}"/> <!--Include hadoop-site.xml for TestFRJoin -->
<pathelement path="${clover.jar}"/>
<path refid="mr-apps-test-ivy.classpath"/>
</path>
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java Mon Oct 15 22:14:46 2012
@@ -163,16 +163,17 @@ public class CSVExcelStorage extends Pig
protected final static byte DOUBLE_QUOTE = '"';
protected final static byte RECORD_DEL = LINEFEED;
- private static byte FIELD_DEL = ',';
- private static String MULTILINE_DEFAULT_STR = "NOMULTILINE";
- private static String LINEBREAKS_DEFAULT_STR = "NOCHANGE";
- private static Multiline MULTILINE_DEFAULT = Multiline.NO;
- private static Linebreaks LINEBREAKS_DEFAULT = Linebreaks.NOCHANGE;
+ private static final byte FIELD_DEL_DEFAULT = ',';
+ private static final String MULTILINE_DEFAULT_STR = "NOMULTILINE";
+ private static final String LINEBREAKS_DEFAULT_STR = "NOCHANGE";
+ private static final Multiline MULTILINE_DEFAULT = Multiline.NO;
+ private static final Linebreaks LINEBREAKS_DEFAULT = Linebreaks.NOCHANGE;
long end = Long.MAX_VALUE;
- Linebreaks eolTreatment = LINEBREAKS_DEFAULT;
- Multiline multilineTreatment = MULTILINE_DEFAULT;
+ private byte fieldDelimiter = FIELD_DEL_DEFAULT;
+ private Linebreaks eolTreatment = LINEBREAKS_DEFAULT;
+ private Multiline multilineTreatment = MULTILINE_DEFAULT;
private ArrayList<Object> mProtoTuple = null;
private TupleFactory mTupleFactory = TupleFactory.getInstance();
@@ -224,7 +225,7 @@ public class CSVExcelStorage extends Pig
*
*/
public CSVExcelStorage() {
- super(new String(new byte[] {FIELD_DEL}));
+ super(new String(new byte[] {FIELD_DEL_DEFAULT}));
}
/**
@@ -297,7 +298,7 @@ public class CSVExcelStorage extends Pig
private void initializeInstance(String delimiter, String multilineStr, String theEofTreatment) {
- FIELD_DEL = StorageUtil.parseFieldDel(delimiter);
+ fieldDelimiter = StorageUtil.parseFieldDel(delimiter);
multilineTreatment = canonicalizeMultilineTreatmentRequest(multilineStr);
eolTreatment = canonicalizeEOLTreatmentRequest(theEofTreatment);
}
@@ -365,7 +366,7 @@ public class CSVExcelStorage extends Pig
// then the entire field must be enclosed in double quotes:
embeddedNewlineIndex = fieldStr.indexOf(LINEFEED);
- if ((fieldStr.indexOf(FIELD_DEL) != -1) ||
+ if ((fieldStr.indexOf(fieldDelimiter) != -1) ||
(fieldStr.indexOf(DOUBLE_QUOTE) != -1) ||
(multilineTreatment == Multiline.YES) && (embeddedNewlineIndex != -1)) {
fieldStr = "\"" + fieldStr + "\"";
@@ -587,7 +588,7 @@ public class CSVExcelStorage extends Pig
}
} else
if (!evenQuotesSeen &&
- (b == FIELD_DEL || b == RECORD_DEL)) {
+ (b == fieldDelimiter || b == RECORD_DEL)) {
getNextInQuotedField = false;
readField(fieldBuffer, getNextFieldID++);
} else {
@@ -604,7 +605,7 @@ public class CSVExcelStorage extends Pig
// that entire field is quoted:
getNextInQuotedField = true;
evenQuotesSeen = true;
- } else if (b == FIELD_DEL) {
+ } else if (b == fieldDelimiter) {
readField(fieldBuffer, getNextFieldID++); // end of the field
} else {
evenQuotesSeen = true;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitionHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitionHelper.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitionHelper.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitionHelper.java Mon Oct 15 22:14:46 2012
@@ -146,9 +146,8 @@ public class TestPathPartitionHelper ext
@Override
protected void setUp() throws Exception {
- File oldConf = new File("build/classes/hadoop-site.xml");
- oldConf.delete();
- conf = new Configuration();
+ conf = new Configuration(false);
+ conf.addResource("core-default.xml");
baseDir = createDir(null,
"testPathPartitioner-testGetKeys-" + System.currentTimeMillis());
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java Mon Oct 15 22:14:46 2012
@@ -54,9 +54,8 @@ public class TestPathPartitioner extends
@Override
protected void setUp() throws Exception {
- File oldConf = new File("build/classes/hadoop-site.xml");
- oldConf.delete();
- conf = new Configuration();
+ conf = new Configuration(false);
+ conf.addResource("core-default.xml");
baseDir = createDir(null,
"testPathPartitioner-testGetKeys-" + System.currentTimeMillis());
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Mon Oct 15 22:14:46 2012
@@ -901,6 +901,7 @@ public class TestAvroStorage {
@Test
public void testFileWithNoExtension() throws IOException {
+ PigSchema2Avro.setTupleIndex(4);
String output= outbasedir + "testFileWithNoExtension";
String expected = basedir + "expected_testFileWithNoExtension.avro";
deleteDirectory(new File(output));
Modified: pig/trunk/src/org/apache/pig/ResourceSchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/ResourceSchema.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/ResourceSchema.java (original)
+++ pig/trunk/src/org/apache/pig/ResourceSchema.java Mon Oct 15 22:14:46 2012
@@ -34,6 +34,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+import org.codehaus.jackson.annotate.JsonPropertyOrder;
/**
* A represenation of a schema used to communicate with load and store functions. This is
@@ -42,6 +43,7 @@ import org.apache.pig.newplan.logical.re
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
+@JsonPropertyOrder({ "fields", "version", "sortKeys", "sortKeyOrders" })
public class ResourceSchema implements Serializable {
private static final long serialVersionUID = 1L;
@@ -65,6 +67,7 @@ public class ResourceSchema implements S
private int version = 0;
+ @JsonPropertyOrder({ "name", "type", "description", "schema" })
public static class ResourceFieldSchema implements Serializable {
private static final long serialVersionUID = 1L;
private String name;
Modified: pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java (original)
+++ pig/trunk/src/org/apache/pig/data/SchemaTupleFrontend.java Mon Oct 15 22:14:46 2012
@@ -213,6 +213,7 @@ public class SchemaTupleFrontend {
*/
public static void reset() {
stf = null;
+ schemasToGenerate.clear();
}
/**
Modified: pig/trunk/test/org/apache/pig/test/TestGFCross.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestGFCross.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestGFCross.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestGFCross.java Mon Oct 15 22:14:46 2012
@@ -39,6 +39,7 @@ public class TestGFCross {
// join groups
@Test
public void testDefault() throws Exception {
+ UDFContext.getUDFContext().addJobConf(null);
Tuple t = TupleFactory.getInstance().newTuple(2);
t.set(0, 2);
Modified: pig/trunk/test/org/apache/pig/test/TestHBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestHBaseStorage.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestHBaseStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestHBaseStorage.java Mon Oct 15 22:14:46 2012
@@ -122,7 +122,7 @@ public class TestHBaseStorage {
// DVR: I've found that it is faster to delete all rows in small tables
// than to drop them.
private void deleteAllRows(String tableName) throws Exception {
- HTable table = new HTable(conf, tableName);
+ HTable table = new HTable(tableName);
ResultScanner scanner = table.getScanner(new Scan());
List<Delete> deletes = Lists.newArrayList();
for (Result row : scanner) {
Modified: pig/trunk/test/org/apache/pig/test/TestJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJobControlCompiler.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJobControlCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJobControlCompiler.java Mon Oct 15 22:14:46 2012
@@ -60,12 +60,36 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.OperatorKey;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestJobControlCompiler {
private static final Configuration CONF = new Configuration();
+
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ // creating a hadoop-site.xml and making it visible to Pig
+ // making sure it is at the same location as for other tests to not pick
+ // up a conf from a previous test
+ File conf_dir = new File("build/classes");
+ File hadoopSite = new File(conf_dir, "hadoop-site.xml");
+ hadoopSite.deleteOnExit();
+ FileWriter fw = new FileWriter(hadoopSite);
+ try {
+ fw.write("<?xml version=\"1.0\"?>\n");
+ fw.write("<?xml-stylesheet type=\"text/xsl\" href=\"nutch-conf.xsl\"?>\n");
+ fw.write("<configuration>\n");
+ fw.write("</configuration>\n");
+ } finally {
+ fw.close();
+ }
+ // making hadoop-site.xml visible to Pig as it REQUIRES!!! one when
+ // running in mapred mode
+ Thread.currentThread().setContextClassLoader(
+ new URLClassLoader(new URL[] { conf_dir.toURI().toURL() }));
+ }
/**
* specifically tests that REGISTERED jars get added to distributed cache instead of merged into
* the job jar
@@ -80,25 +104,6 @@ public class TestJobControlCompiler {
String className = createTestJar(tmpFile);
final String testUDFFileName = className+".class";
- // creating a hadoop-site.xml and making it visible to Pig
- // making sure it is at the same location as for other tests to not pick up a
- // conf from a previous test
- File conf_dir = new File("build/classes");
- File hadoopSite = new File(conf_dir, "hadoop-site.xml");
- hadoopSite.deleteOnExit();
- FileWriter fw = new FileWriter(hadoopSite);
- try {
- fw.write("<?xml version=\"1.0\"?>\n");
- fw.write("<?xml-stylesheet type=\"text/xsl\" href=\"nutch-conf.xsl\"?>\n");
- fw.write("<configuration>\n");
- fw.write("</configuration>\n");
- } finally {
- fw.close();
- }
- // making hadoop-site.xml visible to Pig as it REQUIRES!!! one when running in mapred mode
- Thread.currentThread().setContextClassLoader(
- new URLClassLoader(new URL[] {conf_dir.toURI().toURL()}));
-
// JobControlCompiler setup
PigContext pigContext = new PigContext(ExecType.MAPREDUCE, new Properties());
pigContext.connect();
Modified: pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java Mon Oct 15 22:14:46 2012
@@ -30,14 +30,11 @@ import org.apache.pig.data.Tuple;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-@RunWith(JUnit4.class)
public class TestJsonLoaderStorage {
private static PigServer pigServer;
File jsonFile;
-
+
@BeforeClass
public static void setUp() throws Exception{
removeOutput();
@@ -54,7 +51,7 @@ public class TestJsonLoaderStorage {
}
@Test
- public void testJsonStorage1() throws Exception{
+ public void testJsonLoaderStorage1() throws Exception{
removeOutput();
pigServer.registerScript("test/org/apache/pig/test/data/jsonStorage1.pig");
@@ -63,16 +60,12 @@ public class TestJsonLoaderStorage {
String result = Util.readFile(resultFile);
String expected = Util.readFile(new File("test/org/apache/pig/test/data/jsonStorage1.result"));
- Assert.assertTrue(result.equals(expected));
+ Assert.assertEquals(expected, result);
File schemaFile = new File("jsonStorage1.json/.pig_schema");
result = Util.readFile(schemaFile);
expected = Util.readFile(new File("test/org/apache/pig/test/data/jsonStorage1.schema"));
- Assert.assertTrue(result.equals(expected));
- }
-
- @Test
- public void testJsonLoader1() throws Exception{
+ Assert.assertEquals(expected, result);
File tmpFile = File.createTempFile("tmp", null);
tmpFile.delete();
@@ -80,30 +73,25 @@ public class TestJsonLoaderStorage {
pigServer.registerQuery("a = load 'jsonStorage1.json' using JsonLoader();");
pigServer.store("a", tmpFile.getCanonicalPath());
- String result = Util.readFile(new File(tmpFile.getCanonicalPath()+"/part-m-00000"));
- String expected = Util.readFile(new File("test/org/apache/pig/test/data/jsonStorage1.txt"));
- Assert.assertTrue(result.equals(expected));
- }
+ result = Util.readFile(new File(tmpFile.getCanonicalPath()+"/part-m-00000"));
+ expected = Util.readFile(new File("test/org/apache/pig/test/data/jsonStorage1.txt"));
+ Assert.assertEquals(expected, result);
- @Test
- public void testJsonLoader2() throws Exception{
- File tmpFile = File.createTempFile("tmp", null);
+ tmpFile = File.createTempFile("tmp", null);
tmpFile.delete();
- File schemaFile = new File("test/org/apache/pig/test/data/jsonStorage1.schema");
-
pigServer.registerQuery("a = load 'jsonStorage1.json' using" +
" JsonLoader('a0:int,a1:{(a10:int,a11:chararray)},a2:(a20:double,a21:bytearray),a3:[chararray]');");
pigServer.store("a", tmpFile.getCanonicalPath());
- String result = Util.readFile(new File(tmpFile.getCanonicalPath()+"/part-m-00000"));
- String expected = Util.readFile(new File("test/org/apache/pig/test/data/jsonStorage1.txt"));
- Assert.assertTrue(result.equals(expected));
+ result = Util.readFile(new File(tmpFile.getCanonicalPath()+"/part-m-00000"));
+ expected = Util.readFile(new File("test/org/apache/pig/test/data/jsonStorage1.txt"));
+ Assert.assertEquals(expected, result);
}
@Test
- public void testJsonStorage2() throws Exception{
+ public void testJsonLoaderStorage2() throws Exception{
File inputFile = File.createTempFile("tmp", null);
PrintWriter pw = new PrintWriter(new FileWriter(inputFile));
Modified: pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLogToPhyCompiler.java Mon Oct 15 22:14:46 2012
@@ -17,6 +17,9 @@
*/
package org.apache.pig.test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -39,20 +42,43 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.plan.VisitorException;
-import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
-import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.test.junit.OrderedJUnit4Runner;
+import org.apache.pig.test.junit.OrderedJUnit4Runner.TestOrder;
+import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
/**
*
* To generate golden files, update generate to true
*
*/
-public class TestLogToPhyCompiler extends junit.framework.TestCase {
+
+@RunWith(OrderedJUnit4Runner.class)
+@TestOrder({
+ "testComplexForeach",
+ "testSort",
+ "testDistinct",
+ "testCogroup",
+ "testArithmetic",
+ "testComparison",
+ "testBinCond",
+ "testGenerate",
+ "testUnion",
+ "testSplit",
+ "testIsNull",
+ "testLimit",
+ "testSortInfoAsc",
+ "testSortInfoAscDesc",
+ "testSortInfoNoOrderBy1",
+ "testSortInfoNoOrderBy2",
+ "testSortInfoOrderByLimit",
+ "testSortInfoMultipleStore",
+ "testSortInfoNoOrderBySchema" })
+public class TestLogToPhyCompiler {
File A;
final int MAX_RANGE = 10;
@@ -65,8 +91,8 @@ public class TestLogToPhyCompiler extend
private static final int MAX_SIZE = 100000;;
- @Override
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
pigServer = new PigServer( ExecType.LOCAL, new Properties() );
pc.connect();
}
@@ -145,6 +171,7 @@ public class TestLogToPhyCompiler extend
"testComplexForeach");
}
+ @Test
public void testSort() throws Exception {
checkAgainstGolden(
"store (order (load 'a') by $0) into 'output';",
@@ -153,7 +180,7 @@ public class TestLogToPhyCompiler extend
"testSort");
}
-
+ @Test
public void testDistinct() throws Exception {
checkAgainstGolden(
"store( distinct (load 'a') ) into 'output';",
@@ -161,7 +188,8 @@ public class TestLogToPhyCompiler extend
false,
"testDistinct");
}
-
+
+ @Test
public void testCogroup() throws Exception {
checkAgainstGolden(
"A = cogroup (load 'a') by ($0 + $1, $0 - $1), (load 'b') by ($0 + $1, $0 - $1);" +
@@ -170,7 +198,8 @@ public class TestLogToPhyCompiler extend
false,
"testCogroup");
}
-
+
+ @Test
public void testArithmetic() throws Exception {
checkAgainstGolden(
"A = foreach (load 'A') generate $0 + $1 + 5, $0 - 5 - $1, 'hello';" +
@@ -180,6 +209,7 @@ public class TestLogToPhyCompiler extend
"testArithmetic");
}
+ @Test
public void testComparison() throws Exception {
checkAgainstGolden(
"A = filter (load 'a' using " + PigStorage.class.getName() + "(':')) by $0 + $1 > ($0 - $1) * (4 / 2);" +
Modified: pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Mon Oct 15 22:14:46 2012
@@ -17,6 +17,10 @@
*/
package org.apache.pig.test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -57,10 +61,14 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.test.junit.OrderedJUnit4Runner;
+import org.apache.pig.test.junit.OrderedJUnit4Runner.TestOrder;
import org.apache.pig.test.utils.GenPhyOp;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
/**
* Test cases to test the MRCompiler.
@@ -73,7 +81,39 @@ import org.junit.Test;
* that follow it since the operator keys that will be generated through
* Random will be different.
*/
-public class TestMRCompiler extends junit.framework.TestCase {
+
+@RunWith(OrderedJUnit4Runner.class)
+@TestOrder({
+ "testRun1",
+ "testRun2",
+ "testSpl1",
+ "testSpl2",
+ "testSpl3",
+ "testSim1",
+ "testSim2",
+ "testSim3",
+ "testSim5",
+ "testSim6",
+ "testSim7",
+ "testSim8",
+ "testSim9",
+ "testSortUDF1",
+ "testDistinct1",
+ "testLimit",
+ "testMRCompilerErr",
+ "testMRCompilerErr1",
+ "testNumReducersInLimit",
+ "testNumReducersInLimitWithParallel",
+ "testUDFInJoin",
+ "testMergeJoin",
+ "testMergeJoinWithIndexableLoadFunc",
+ "testCastFuncShipped",
+ "testLimitAdjusterFuncShipped",
+ "testSortedDistinctInForeach",
+ "testUDFInMergedCoGroup",
+ "testUDFInMergedJoin",
+ "testSchemaInStoreForDistinctLimit" })
+public class TestMRCompiler {
static MiniCluster cluster = MiniCluster.buildCluster();
static PigContext pc;
@@ -106,7 +146,6 @@ public class TestMRCompiler extends juni
// and are sure of
private boolean generate = false;
- @Override
@Before
public void setUp() throws ExecException {
GenPhyOp.setR(r);
@@ -117,7 +156,6 @@ public class TestMRCompiler extends juni
pigServerMR = new PigServer( pcMR );
}
- @Override
@After
public void tearDown() throws Exception {
}
@@ -231,6 +269,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC10.gld");
}
+ @Test
public void testRun2() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -322,6 +361,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC11.gld");
}
+ @Test
public void testSpl1() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -361,6 +401,7 @@ public class TestMRCompiler extends juni
}
+ @Test
public void testSpl2() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -409,6 +450,7 @@ public class TestMRCompiler extends juni
}
+ @Test
public void testSpl3() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -490,6 +532,7 @@ public class TestMRCompiler extends juni
// Tests Single input case for both blocking and non-blocking
// with both map and reduce phases
+ @Test
public void testSim1() throws Exception {
PhysicalPlan php = new PhysicalPlan();
POLoad ld = GenPhyOp.topLoadOp();
@@ -520,6 +563,7 @@ public class TestMRCompiler extends juni
}
+ @Test
public void testSim2() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -540,6 +584,7 @@ public class TestMRCompiler extends juni
}
+ @Test
public void testSim3() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -592,6 +637,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC4.gld");
}
+ @Test
public void testSim5() throws Exception {
PhysicalPlan php = new PhysicalPlan();
PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
@@ -609,6 +655,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC5.gld");
}
+ @Test
public void testSim6() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -636,6 +683,7 @@ public class TestMRCompiler extends juni
}
+ @Test
public void testSim7() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -667,6 +715,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC7.gld");
}
+ @Test
public void testSim8() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -702,6 +751,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC8.gld");
}
+ @Test
public void testSim9() throws Exception {
PhysicalPlan php = new PhysicalPlan();
@@ -725,6 +775,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC9.gld");
}
+ @Test
public void testSortUDF1() throws Exception {
PhysicalPlan php = new PhysicalPlan();
PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
@@ -791,6 +842,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC15.gld");
}
+ @Test
public void testDistinct1() throws Exception {
PhysicalPlan php = new PhysicalPlan();
PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
@@ -814,7 +866,7 @@ public class TestMRCompiler extends juni
run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC16.gld");
}
-
+ @Test
public void testLimit() throws Exception {
PhysicalPlan php = new PhysicalPlan();
Modified: pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java Mon Oct 15 22:14:46 2012
@@ -24,7 +24,10 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -168,13 +171,17 @@ public class TestMapSideCogroup {
assertEquals(2,mrPlan.size());
Iterator<MapReduceOper> itr = mrPlan.iterator();
- MapReduceOper oper = itr.next();
- assertTrue(oper.reducePlan.isEmpty());
- assertFalse(oper.mapPlan.isEmpty());
-
- oper = itr.next();
- assertFalse(oper.reducePlan.isEmpty());
- assertFalse(oper.mapPlan.isEmpty());
+ List<MapReduceOper> opers = new ArrayList<MapReduceOper>();
+ opers.add(itr.next());
+ opers.add(itr.next());
+ //Order of entrySet is not guaranteed with jdk1.7
+ Collections.sort(opers);
+
+ assertTrue(opers.get(0).reducePlan.isEmpty());
+ assertFalse(opers.get(0).mapPlan.isEmpty());
+
+ assertFalse(opers.get(1).reducePlan.isEmpty());
+ assertFalse(opers.get(1).mapPlan.isEmpty());
} catch(Exception e){
e.printStackTrace();
fail("Compilation of merged cogroup failed.");
Modified: pig/trunk/test/org/apache/pig/test/TestMergeJoinOuter.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMergeJoinOuter.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMergeJoinOuter.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMergeJoinOuter.java Mon Oct 15 22:14:46 2012
@@ -24,7 +24,10 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.Properties;
import org.apache.pig.ExecType;
@@ -118,13 +121,17 @@ public class TestMergeJoinOuter {
assertEquals(2,mrPlan.size());
Iterator<MapReduceOper> itr = mrPlan.iterator();
- MapReduceOper oper = itr.next();
- assertTrue(oper.reducePlan.isEmpty());
- assertFalse(oper.mapPlan.isEmpty());
-
- oper = itr.next();
- assertFalse(oper.reducePlan.isEmpty());
- assertFalse(oper.mapPlan.isEmpty());
+ List<MapReduceOper> opers = new ArrayList<MapReduceOper>();
+ opers.add(itr.next());
+ opers.add(itr.next());
+ //Order of entrySet is not guaranteed with jdk1.7
+ Collections.sort(opers);
+
+ assertTrue(opers.get(0).reducePlan.isEmpty());
+ assertFalse(opers.get(0).mapPlan.isEmpty());
+
+ assertFalse(opers.get(1).reducePlan.isEmpty());
+ assertFalse(opers.get(1).mapPlan.isEmpty());
} catch(Exception e){
Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanLogToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanLogToPhyTranslationVisitor.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPlanLogToPhyTranslationVisitor.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPlanLogToPhyTranslationVisitor.java Mon Oct 15 22:14:46 2012
@@ -17,12 +17,15 @@
*/
package org.apache.pig.test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import java.util.Properties;
-import junit.framework.TestCase;
-
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
@@ -77,8 +80,10 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+import org.junit.Before;
+import org.junit.Test;
-public class TestNewPlanLogToPhyTranslationVisitor extends TestCase {
+public class TestNewPlanLogToPhyTranslationVisitor {
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
private PhysicalPlan translatePlan(OperatorPlan plan) throws IOException {
@@ -93,10 +98,12 @@ public class TestNewPlanLogToPhyTranslat
return Util.buildLp(pigServer, query);
}
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
LogicalExpression.resetNextUid();
}
-
+
+ @Test
public void testSimplePlan() throws Exception {
String query = ("a = load 'd.txt';" +
"b = filter a by $0==NULL;" +
@@ -117,16 +124,19 @@ public class TestNewPlanLogToPhyTranslat
PhysicalOperator fil = phyPlan.getSuccessors(load).get(0);
assertEquals( POFilter.class, fil.getClass() );
PhysicalPlan filPlan = ((POFilter)fil).getPlan();
- assertEquals( 2, filPlan.getRoots().size() );
+ List<PhysicalOperator> roots = filPlan.getRoots();
+ // Order of entrySet is not guaranteed with jdk1.7
+ Collections.sort(roots);
+ assertEquals( 2, roots.size() );
assertEquals( 1, filPlan.getLeaves().size() );
PhysicalOperator eq = filPlan.getLeaves().get(0);
assertEquals( EqualToExpr.class, eq.getClass() );
- PhysicalOperator prj1 = filPlan.getRoots().get(1);
+ PhysicalOperator prj1 = roots.get(0);
assertEquals( POProject.class, prj1.getClass() );
assertEquals( 0, ((POProject)prj1).getColumn() );
- PhysicalOperator constExp = filPlan.getRoots().get(0);
+ PhysicalOperator constExp = roots.get(1);
assertEquals( ConstantExpression.class, constExp.getClass() );
assertEquals( null, ((ConstantExpression)constExp).getValue() );
@@ -135,7 +145,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POStore.class, stor.getClass() );
assertTrue( ((POStore)stor).getSFile().getFileName().contains("empty"));
}
-
+
+ @Test
public void testJoinPlan() throws Exception {
String query = ("a = load 'd1.txt' as (id, c);" +
"b = load 'd2.txt'as (id, c);" +
@@ -147,10 +158,12 @@ public class TestNewPlanLogToPhyTranslat
LogicalPlan newPlan = buildPlan(query);
PhysicalPlan physicalPlan = translatePlan(newPlan);
assertEquals(9, physicalPlan.size());
- assertEquals(physicalPlan.getRoots().size(), 2);
+ List<PhysicalOperator> roots = physicalPlan.getRoots();
+ Collections.sort(roots);
+ assertEquals(roots.size(), 2);
// Check Load and LocalRearrange and GlobalRearrange
- PhysicalOperator LoR = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(1)).get(0);
+ PhysicalOperator LoR = (PhysicalOperator)physicalPlan.getSuccessors(roots.get(0)).get(0);
assertEquals( POLocalRearrange.class, LoR.getClass() );
POLocalRearrange Lor = (POLocalRearrange) LoR;
PhysicalOperator prj3 = Lor.getPlans().get(0).getLeaves().get(0);
@@ -160,7 +173,7 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POLoad.class, inp1.getClass() );
assertTrue( ((POLoad)inp1).getLFile().getFileName().contains("d1.txt") );
- PhysicalOperator LoR1 = (PhysicalOperator)physicalPlan.getSuccessors(physicalPlan.getRoots().get(0)).get(0);
+ PhysicalOperator LoR1 = (PhysicalOperator)physicalPlan.getSuccessors(roots.get(1)).get(0);
assertEquals( POLocalRearrange.class, LoR1.getClass() );
POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
PhysicalOperator prj4 = Lor1.getPlans().get(0).getLeaves().get(0);
@@ -192,9 +205,11 @@ public class TestNewPlanLogToPhyTranslat
PhysicalPlan filPlan = ((POFilter)fil).getPlan();
List<PhysicalOperator> filRoots = filPlan.getRoots();
+ assertEquals(4, filRoots.size());
+ Collections.sort(filRoots);
- assertEquals( ConstantExpression.class, filRoots.get(2).getClass() );
- ConstantExpression ce1 = (ConstantExpression) filRoots.get(2);
+ assertEquals( ConstantExpression.class, filRoots.get(1).getClass() );
+ ConstantExpression ce1 = (ConstantExpression) filRoots.get(1);
assertEquals( null, ce1.getValue() );
assertEquals( ConstantExpression.class, filRoots.get(3).getClass() );
ConstantExpression ce2 = (ConstantExpression) filRoots.get(3);
@@ -202,8 +217,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, filRoots.get(0).getClass() );
POProject prj1 = (POProject) filRoots.get(0);
assertEquals( 0, prj1.getColumn() );
- assertEquals( POProject.class, filRoots.get(1).getClass() );
- POProject prj2 = (POProject) filRoots.get(1);
+ assertEquals( POProject.class, filRoots.get(2).getClass() );
+ POProject prj2 = (POProject) filRoots.get(2);
assertEquals( 3, prj2.getColumn() );
@@ -212,7 +227,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POStore.class, stor.getClass() );
assertTrue( ((POStore)stor).getSFile().getFileName().contains("empty") );
}
-
+
+ @Test
public void testMultiStore() throws Exception {
String query = ("a = load 'd1.txt' as (id, c);" +
"b = load 'd2.txt'as (id, c);" +
@@ -227,11 +243,13 @@ public class TestNewPlanLogToPhyTranslat
LogicalPlan newLogicalPlan = buildPlan(query);
PhysicalPlan phyPlan = translatePlan(newLogicalPlan);
assertEquals(16, phyPlan.size());
- assertEquals(phyPlan.getRoots().size(), 3);
+ List<PhysicalOperator> phyPlanRoots = phyPlan.getRoots();
+ Collections.sort(phyPlanRoots);
+ assertEquals(phyPlanRoots.size(), 3);
assertEquals(phyPlan.getLeaves().size(), 1 );
// Check Load and LocalRearrange and GlobalRearrange
- PhysicalOperator LoR = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(0)).get(0);
+ PhysicalOperator LoR = (PhysicalOperator)phyPlan.getSuccessors(phyPlanRoots.get(2)).get(0);
assertEquals( POLocalRearrange.class, LoR.getClass() );
POLocalRearrange Lor = (POLocalRearrange) LoR;
PhysicalOperator prj1 = Lor.getPlans().get(0).getLeaves().get(0);
@@ -241,7 +259,7 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POLoad.class, inp1.getClass() );
assertTrue( ((POLoad)inp1).getLFile().getFileName().contains("d3.txt") );
- PhysicalOperator LoR1 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(1)).get(0);
+ PhysicalOperator LoR1 = (PhysicalOperator)phyPlan.getSuccessors(phyPlanRoots.get(0)).get(0);
assertEquals( POLocalRearrange.class, LoR1.getClass() );
POLocalRearrange Lor1 = (POLocalRearrange) LoR1;
PhysicalOperator prj2 = Lor1.getPlans().get(0).getLeaves().get(0);
@@ -257,7 +275,7 @@ public class TestNewPlanLogToPhyTranslat
PhysicalOperator Pack = (PhysicalOperator)phyPlan.getSuccessors(GoR).get(0);
assertEquals( POPackage.class, Pack.getClass() );
- PhysicalOperator LoR2 = (PhysicalOperator)phyPlan.getSuccessors(phyPlan.getRoots().get(2)).get(0);
+ PhysicalOperator LoR2 = (PhysicalOperator)phyPlan.getSuccessors(phyPlanRoots.get(1)).get(0);
assertEquals( POLocalRearrange.class, LoR2.getClass() );
POLocalRearrange Lor2 = (POLocalRearrange) LoR2;
PhysicalOperator prj3 = Lor2.getPlans().get(0).getLeaves().get(0);
@@ -298,18 +316,19 @@ public class TestNewPlanLogToPhyTranslat
PhysicalPlan filPlan = ((POFilter)fil).getPlan();
List<PhysicalOperator> filRoots = filPlan.getRoots();
+ Collections.sort(filRoots);
- assertEquals( ConstantExpression.class, filRoots.get(0).getClass() );
- ConstantExpression ce1 = (ConstantExpression) filRoots.get(0);
+ assertEquals( ConstantExpression.class, filRoots.get(1).getClass() );
+ ConstantExpression ce1 = (ConstantExpression) filRoots.get(1);
assertEquals( null, ce1.getValue() );
- assertEquals( ConstantExpression.class, filRoots.get(2).getClass() );
- ConstantExpression ce2 = (ConstantExpression) filRoots.get(2);
+ assertEquals( ConstantExpression.class, filRoots.get(3).getClass() );
+ ConstantExpression ce2 = (ConstantExpression) filRoots.get(3);
assertEquals( null, ce2.getValue() );
- assertEquals( POProject.class, filRoots.get(1).getClass() );
- POProject prj8 = (POProject) filRoots.get(1);
+ assertEquals( POProject.class, filRoots.get(2).getClass() );
+ POProject prj8 = (POProject) filRoots.get(2);
assertEquals( 5, prj8.getColumn() );
- assertEquals( POProject.class, filRoots.get(3).getClass() );
- POProject prj9 = (POProject) filRoots.get(3);
+ assertEquals( POProject.class, filRoots.get(0).getClass() );
+ POProject prj9 = (POProject) filRoots.get(0);
assertEquals( 2, prj9.getColumn() );
@@ -325,18 +344,19 @@ public class TestNewPlanLogToPhyTranslat
PhysicalPlan filPlan2 = ((POFilter)fil2).getPlan();
List<PhysicalOperator> filRoots2 = filPlan2.getRoots();
+ Collections.sort(filRoots2);
- assertEquals( ConstantExpression.class, filRoots2.get(0).getClass() );
- ConstantExpression ce3 = (ConstantExpression) filRoots2.get(0);
+ assertEquals( ConstantExpression.class, filRoots2.get(1).getClass() );
+ ConstantExpression ce3 = (ConstantExpression) filRoots2.get(1);
assertEquals( null, ce3.getValue() );
- assertEquals( ConstantExpression.class, filRoots2.get(2).getClass() );
- ConstantExpression ce4 = (ConstantExpression) filRoots2.get(2);
+ assertEquals( ConstantExpression.class, filRoots2.get(3).getClass() );
+ ConstantExpression ce4 = (ConstantExpression) filRoots2.get(3);
assertEquals( null, ce4.getValue() );
- assertEquals( POProject.class, filRoots2.get(1).getClass() );
- POProject prj10 = (POProject) filRoots2.get(1);
+ assertEquals( POProject.class, filRoots2.get(2).getClass() );
+ POProject prj10 = (POProject) filRoots2.get(2);
assertEquals( 3, prj10.getColumn() );
- assertEquals( POProject.class, filRoots2.get(3).getClass() );
- POProject prj11 = (POProject) filRoots2.get(3);
+ assertEquals( POProject.class, filRoots2.get(0).getClass() );
+ POProject prj11 = (POProject) filRoots2.get(0);
assertEquals( 0, prj11.getColumn() );
// Check Store Operator
@@ -345,7 +365,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POStore.class, stor.getClass() );
assertTrue( ((POStore)stor).getSFile().getFileName().contains("empty") );
}
-
+
+ @Test
public void testPlanWithCast() throws Exception {
String query = ("a = load 'd.txt' as (id, c);" +
"b = filter a by (int)id==10;" +
@@ -386,7 +407,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POStore.class, stor.getClass() );
assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );
}
-
+
+ @Test
public void testPlanWithGreaterThan() throws Exception {
String query = ("a = load 'd.txt' as (id, c);" +
"b = filter a by (int)id>10;" +
@@ -427,7 +449,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POStore.class, stor.getClass() );
assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );
}
-
+
+ @Test
public void testPlanWithLessThan() throws Exception {
String query = ("a = load 'd.txt' as (id, c);" +
"b = filter a by (int)id<10;" +
@@ -469,6 +492,7 @@ public class TestNewPlanLogToPhyTranslat
assertTrue( ((POStore)stor).getSFile().getFileName().contains( "empty" ) );
}
+ @Test
public void testForeachPlan() throws Exception {
String query = ("a = load 'd.txt' as (id, c);" +
"b = foreach a generate id, c;" +
@@ -498,7 +522,8 @@ public class TestNewPlanLogToPhyTranslat
assertFalse(flat[0]);
assertFalse(flat[1]);
}
-
+
+ @Test
public void testForeachPlan2() throws Exception {
String query = ("a = load 'd.txt' as (id, c:bag{t:(s,v)});" +
"b = foreach a generate id, flatten(c);" +
@@ -542,7 +567,8 @@ public class TestNewPlanLogToPhyTranslat
assertFalse(flat[0]);
assertTrue(flat[1]);
}
-
+
+ @Test
public void testPlanwithPlus() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate a+b;" +
@@ -596,7 +622,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, pAdd.getLhs().getClass() );
assertEquals( POProject.class, pAdd.getRhs().getClass() );
}
-
+
+ @Test
public void testPlanwithSubtract() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate a-b;" +
@@ -649,7 +676,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, pSubtract.getLhs().getClass() );
assertEquals( POProject.class, pSubtract.getRhs().getClass() );
}
-
+
+ @Test
public void testPlanwithMultiply() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate a*b;" +
@@ -702,7 +730,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, pMultiply.getLhs().getClass() );
assertEquals( POProject.class, pMultiply.getRhs().getClass() );
}
-
+
+ @Test
public void testPlanwithDivide() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate a/b;" +
@@ -755,7 +784,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, pDivide.getLhs().getClass() );
assertEquals( POProject.class, pDivide.getRhs().getClass() );
}
-
+
+ @Test
public void testPlanwithMod() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate a%b;" +
@@ -808,7 +838,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, pMod.getLhs().getClass() );
assertEquals( POProject.class, pMod.getRhs().getClass() );
}
-
+
+ @Test
public void testPlanwithNegative() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate -a;" +
@@ -859,7 +890,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( 1, inputPln.getRoots().size() );
assertEquals( POProject.class, pNegative.getInputs().get(0).getClass() );
}
-
+
+ @Test
public void testPlanwithisNull() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = filter a by a is null;" +
@@ -894,7 +926,8 @@ public class TestNewPlanLogToPhyTranslat
ProjectExpression prj = (ProjectExpression) isNull.getExpression();
assertEquals( ls.getField(0).uid, prj.getFieldSchema().uid );
}
-
+
+ @Test
public void testPlanwithisNotNull() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = filter a by a is not null;" +
@@ -933,7 +966,8 @@ public class TestNewPlanLogToPhyTranslat
ProjectExpression prj = (ProjectExpression) isNull.getExpression();
assertEquals( ls.getField(0).uid, prj.getFieldSchema().uid );
}
-
+
+ @Test
public void testPlanwithBinCond() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:int);" +
"b = foreach a generate ( a < b ? b : a );" +
@@ -1014,7 +1048,8 @@ public class TestNewPlanLogToPhyTranslat
POProject prj_4 = (POProject) lessThan_p.getRhs();
assertEquals( 1, prj_4.getColumn() );
}
-
+
+ @Test
public void testPlanwithUserFunc() throws Exception {
String query = ("a = load 'd.txt' as (a:int, b:bag{t:tuple(b_a:int,b_b:int)});" +
"b = foreach a generate a,COUNT(b);" +
@@ -1074,7 +1109,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( 1, ((POProject)inputPln2.getRoots().get(0)).getColumn() );
}
-
+
+ @Test
public void testPlanwithUserFunc2() throws Exception {
// This one uses BagDereferenceExpression
String query = ("a = load 'd.txt' as (a:int, b:bag{t:tuple(b_a:int,b_b:int)});" +
@@ -1141,7 +1177,8 @@ public class TestNewPlanLogToPhyTranslat
assertEquals( POProject.class, inputPln2.getSuccessors(prj3).get(0).getClass() );
assertEquals( 0, ((POProject)inputPln2.getSuccessors(prj3).get(0)).getColumn() );
}
-
+
+ @Test
public void testCogroup() throws Exception {
String query = ("a = load 'd.txt' as (name:chararray, age:int, gpa:float);" +
"b = group a by name;" +
@@ -1174,7 +1211,8 @@ public class TestNewPlanLogToPhyTranslat
POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
assertEquals( DataType.TUPLE, pack.getResultType() );
}
-
+
+ @Test
public void testCogroup2() throws Exception {
String query = ("a = load 'd.txt' as (name:chararray, age:int, gpa:float);" +
"b = group a by ( name, age );" +
@@ -1212,7 +1250,8 @@ public class TestNewPlanLogToPhyTranslat
POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
assertEquals( DataType.TUPLE, pack.getResultType() );
}
-
+
+ @Test
public void testCogroup3() throws Exception {
String query = "a = load 'd.txt' as (name:chararray, age:int, gpa:float);" +
"b = load 'e.txt' as (name:chararray, age:int, gpa:float);" +
@@ -1256,7 +1295,8 @@ public class TestNewPlanLogToPhyTranslat
POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
assertEquals( DataType.TUPLE, pack.getResultType() );
}
-
+
+ @Test
public void testCogroup4() throws Exception {
String query = "a = load 'd.txt' as (name:chararray, age:int, gpa:float);" +
"b = load 'e.txt' as (name:chararray, age:int, gpa:float);" +
@@ -1313,7 +1353,8 @@ public class TestNewPlanLogToPhyTranslat
POPackage pack = (POPackage)phyPlan.getSuccessors(globalR).get(0);
assertEquals( DataType.TUPLE, pack.getResultType() );
}
-
+
+ @Test
public void testUserDefinedForEachSchema1() throws Exception {
String query = ("a = load 'a.txt';" +
"b = foreach a generate $0 as a0, $1 as a1;" +
@@ -1329,6 +1370,7 @@ public class TestNewPlanLogToPhyTranslat
assertTrue(foreach.getSchema().getField(1).alias.equals("a1"));
}
+ @Test
public void testUserDefinedForEachSchema2() throws Exception {
String query = ("a = load 'a.txt' as (b:bag{});" +
"b = foreach a generate flatten($0) as (a0, a1);" +
@@ -1343,7 +1385,8 @@ public class TestNewPlanLogToPhyTranslat
assertTrue(foreach.getSchema().getField(0).alias.equals("a0"));
assertTrue(foreach.getSchema().getField(1).alias.equals("a1"));
}
-
+
+ @Test
// See PIG-767
public void testCogroupSchema1() throws Exception {
String query = ("a = load '1.txt' as (a0, a1);" +
@@ -1363,7 +1406,8 @@ public class TestNewPlanLogToPhyTranslat
LogicalSchema tupleSchema = bagSchema.getField(0).schema;
assertEquals(tupleSchema.size(), 2);
}
-
+
+ @Test
// See PIG-767
public void testCogroupSchema2() throws Exception {
String query = "a = load '1.txt' as (a0, a1);" +
Modified: pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java?rev=1398536&r1=1398535&r2=1398536&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java Mon Oct 15 22:14:46 2012
@@ -92,7 +92,8 @@ public class TestScriptLanguage {
"2\t4",
"3\t5"
};
-
+
+ Util.deleteFile(cluster, "simple_table");
Util.createInputFile(cluster, "simple_table", input);
Util.createLocalInputFile( "testScript.py", script);