You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/11/24 22:36:26 UTC
svn commit: r883886 [3/3] -
/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPONegative.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPONegative.java?rev=883886&r1=883885&r2=883886&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPONegative.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestPONegative.java Tue Nov 24 21:36:25 2009
@@ -34,8 +34,6 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONegative;
import org.apache.pig.impl.plan.PlanException;
-import org.junit.Before;
-
import junit.framework.TestCase;
public class TestPONegative extends TestCase {
@@ -69,11 +67,9 @@
int output = (Integer) pn.getNext(expected).result;
assertEquals(expected.intValue(), output);
}
-
}
public void testPONegIntAndNull () throws PlanException, ExecException {
-
for(int i = 0; i < MAX; i++) {
Tuple t = tf.newTuple();
t.append(r.nextInt());
@@ -83,7 +79,6 @@
t.append(null);
bag.add(t);
}
-
}
POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
@@ -107,11 +102,8 @@
Integer expected = -(Integer)t.get(0);
int output = (Integer) pn.getNext(expected).result;
assertEquals(expected.intValue(), output);
-
}
-
}
-
}
public void testPONegLong () throws PlanException, ExecException {
@@ -137,7 +129,6 @@
long output = (Long) pn.getNext(expected).result;
assertEquals(expected.longValue(), output);
}
-
}
public void testPONegLongAndNull () throws PlanException, ExecException {
@@ -150,7 +141,6 @@
t.append(null);
bag.add(t);
}
-
}
POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
@@ -174,12 +164,8 @@
Long expected = -(Long)t.get(0);
long output = (Long) pn.getNext(expected).result;
assertEquals(expected.longValue(), output);
-
}
-
-
}
-
}
public void testPONegDouble() throws PlanException, ExecException {
@@ -204,12 +190,9 @@
Double expected = -(Double)t.get(0);
double output = (Double) pn.getNext(expected).result;
assertEquals(expected.doubleValue(), output);
-
}
-
}
-
public void testPONegDoubleAndNull() throws PlanException, ExecException {
for(int i = 0; i < MAX; i++) {
Tuple t = tf.newTuple();
@@ -220,7 +203,6 @@
t.append(null);
bag.add(t);
}
-
}
POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
@@ -239,16 +221,12 @@
if(t.get(0) == null) {
Double output = (Double )pn.getNext((Double )null).result;
assertEquals(null, output);
-
} else {
Double expected = -(Double)t.get(0);
double output = (Double) pn.getNext(expected).result;
assertEquals(expected.doubleValue(), output);
-
}
-
}
-
}
public void testPONegFloat() throws PlanException, ExecException {
@@ -274,10 +252,8 @@
float output = (Float) pn.getNext(expected).result;
assertEquals(expected.floatValue(), output);
}
-
}
-
public void testPONegFloatAndNull() throws PlanException, ExecException {
for(int i = 0; i < MAX; i++) {
Tuple t = tf.newTuple();
@@ -288,7 +264,6 @@
t.append(null);
bag.add(t);
}
-
}
POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
@@ -307,22 +282,19 @@
if(t.get(0) == null) {
Float output = (Float)pn.getNext((Float)null).result;
assertEquals(null, output);
-
} else {
-
Float expected = -(Float)t.get(0);
float output = (Float) pn.getNext(expected).result;
assertEquals(expected.floatValue(), output);
-
}
}
-
}
public void testPONegType() throws Exception {
PigServer pig = new PigServer(ExecType.MAPREDUCE, miniCluster.getProperties());
File f = Util.createInputFile("tmp", "", new String[] {"a", "b", "c"});
- pig.registerQuery("a = load '" + Util.generateURI(f.toString()) + "';");
+ pig.registerQuery("a = load '"
+ + Util.generateURI(f.toString(), pig.getPigContext()) + "';");
// -1 is modeled as POnegative with Constant(1)
pig.registerQuery("b = foreach a generate SIZE(-1);");
Iterator<Tuple> it = pig.openIterator("b");
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSampleOptimizer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSampleOptimizer.java?rev=883886&r1=883885&r2=883886&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSampleOptimizer.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSampleOptimizer.java Tue Nov 24 21:36:25 2009
@@ -165,7 +165,9 @@
}
ps.close();
- pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "' using PigStorage() AS (num:int);");
+ pigServer.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pigServer.getPigContext())
+ + "' using PigStorage() AS (num:int);");
pigServer.registerQuery("B = order A by num desc;");
Iterator<Tuple> result = pigServer.openIterator("B");
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSplitStore.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSplitStore.java?rev=883886&r1=883885&r2=883886&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSplitStore.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSplitStore.java Tue Nov 24 21:36:25 2009
@@ -62,7 +62,8 @@
@Test
public void test1() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.store("A1", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
pig.store("A2", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
@@ -70,7 +71,8 @@
@Test
public void test2() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A1");
pig.store("A2", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
@@ -78,7 +80,8 @@
@Test
public void test3() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A2");
pig.store("A1", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
@@ -86,7 +89,8 @@
@Test
public void test4() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.store("A1", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
pig.openIterator("A2");
@@ -94,7 +98,8 @@
@Test
public void test5() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.store("A2", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
pig.openIterator("A1");
@@ -102,7 +107,8 @@
@Test
public void test6() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A1");
pig.registerQuery("Store A2 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
@@ -110,7 +116,8 @@
@Test
public void test7() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A2");
pig.registerQuery("Store A1 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
@@ -118,7 +125,8 @@
@Test
public void test8() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.registerQuery("Store A1 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
pig.openIterator("A2");
@@ -126,7 +134,8 @@
@Test
public void test9() throws Exception{
- pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pig.registerQuery("A = LOAD '"
+ + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.registerQuery("Store A2 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
pig.openIterator("A1");
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestUTF8.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestUTF8.java?rev=883886&r1=883885&r2=883886&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestUTF8.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestUTF8.java Tue Nov 24 21:36:25 2009
@@ -60,7 +60,9 @@
pw.println("ภาษาà¹à¸à¸¢");
pw.close();
- pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString()) + "' using " + PigStorage.class.getName() + "();");
+ pigServer.registerQuery("a = load '"
+ + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ + "' using " + PigStorage.class.getName() + "();");
Iterator<Tuple> iter = pigServer.openIterator("a");
assertEquals(DataType.toString(iter.next().get(0)), "ä¸æ");
@@ -91,7 +93,9 @@
pw.println("ä¸æ");
pw.close();
- pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString()) + "' using " + PigStorage.class.getName() + "();");
+ pigServer.registerQuery("a = load '"
+ + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ + "' using " + PigStorage.class.getName() + "();");
pigServer.registerQuery("b = filter a by $0 == 'ä¸æ';");
Iterator<Tuple> iter = pigServer.openIterator("a");
@@ -99,6 +103,7 @@
f1.delete();
}
+
@Test
public void testParamSubstitution() throws Exception{
File queryFile = File.createTempFile("query", "");
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java?rev=883886&r1=883885&r2=883886&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java Tue Nov 24 21:36:25 2009
@@ -45,6 +45,7 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
@@ -58,6 +59,7 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.parser.QueryParser;
@@ -391,6 +393,17 @@
return "file:"+path;
}
+ static String generateURI(String filename, PigContext context)
+ throws IOException {
+ if (context.getExecType() == ExecType.MAPREDUCE) {
+ return FileLocalizer.hadoopify(filename, context);
+ } else if (context.getExecType() == ExecType.LOCAL) {
+ return "file://" + filename;
+ } else {
+ throw new IllegalStateException("ExecType: " + context.getExecType());
+ }
+ }
+
public static Schema getSchemaFromString(String schemaString) throws ParseException {
return Util.getSchemaFromString(schemaString, DataType.BYTEARRAY);
}