You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/05/30 20:50:07 UTC
svn commit: r1487970 - in /pig/trunk: ./
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
Author: daijy
Date: Thu May 30 18:50:07 2013
New Revision: 1487970
URL: http://svn.apache.org/r1487970
Log:
PIG-3334: Fix Windows piggybank unit test failures
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestAllLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCombinedLogLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCommonLogLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestDBStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLoadFuncHelper.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMyRegExLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestSequenceFileLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 30 18:50:07 2013
@@ -192,6 +192,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3334: Fix Windows piggybank unit test failures (daijy)
+
PIG-3337: Fix remaining Window e2e tests (daijy)
PIG-3328: DataBags created with an initial list of tuples don't get registered as spillable (mwagner via daijy)
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestAllLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestAllLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestAllLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestAllLoader.java Thu May 30 18:50:07 2013
@@ -52,6 +52,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.piggybank.storage.AllLoader;
import org.apache.pig.piggybank.storage.HiveColumnarLoader;
import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper;
+import org.apache.pig.test.Util;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -159,7 +160,7 @@ public class TestAllLoader extends TestC
server.setBatchOn();
server.registerFunction(allLoaderName, new FuncSpec(allLoaderFuncSpec));
- server.registerQuery("a = LOAD '" + filesByContentDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(filesByContentDir.getAbsolutePath())
+ "' using " + allLoaderFuncSpec + " as (p:float, q:float);");
readRecordsFromLoader(server, "a", fileRecords);
@@ -190,7 +191,7 @@ public class TestAllLoader extends TestC
String schema = taggedSchemas[i++];
server.registerQuery(tag + " = LOAD '"
- + taggedLogicPartitionDir.getAbsolutePath() + "/" + tag
+ + Util.encodeEscape(taggedLogicPartitionDir.getAbsolutePath()) + "/" + tag
+ "' using " + allLoaderFuncSpec + " as " + schema + ";");
readRecordsFromLoader(server, tag, fileRecords
@@ -208,7 +209,7 @@ public class TestAllLoader extends TestC
@Test
public void testLogicPartitionFilter() throws IOException {
- server.registerQuery("a = LOAD '" + logicPartitionDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(logicPartitionDir.getAbsolutePath())
+ "' using " + allLoaderName + "('block<=2')"
+ " as (q:float, p:float);");
@@ -239,7 +240,7 @@ public class TestAllLoader extends TestC
@Test
public void testLogicPartitionPartitionColumnExtract() throws IOException {
- server.registerQuery("a = LOAD '" + logicPartitionDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(logicPartitionDir.getAbsolutePath())
+ "' using " + allLoaderFuncSpec
+ " as (q:float, p:float, block:chararray);");
@@ -285,7 +286,7 @@ public class TestAllLoader extends TestC
@Test
public void testLogicPartitionDir() throws IOException {
- server.registerQuery("a = LOAD '" + logicPartitionDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(logicPartitionDir.getAbsolutePath())
+ "' using " + allLoaderFuncSpec + " as (q:float, p:float);");
readRecordsFromLoader(server, "a", fileRecords * logicPartitions.length
@@ -302,7 +303,7 @@ public class TestAllLoader extends TestC
public void testDateParitionFilterWithAsSchema() throws IOException {
server.registerQuery("a = LOAD '"
- + datePartitionDir.getAbsolutePath()
+ + Util.encodeEscape(datePartitionDir.getAbsolutePath())
+ "' using "
+ allLoaderName
+ "('daydate >= \"2010-11-02\" and daydate <= \"2010-11-04\"') AS (q:float, p:float, daydate:chararray); ");
@@ -333,7 +334,7 @@ public class TestAllLoader extends TestC
public void testDateParitionFilterWithoutSchema() throws IOException {
server.registerQuery("a = LOAD '"
- + datePartitionDir.getAbsolutePath()
+ + Util.encodeEscape(datePartitionDir.getAbsolutePath())
+ "' using "
+ allLoaderName
+ "('daydate >= \"2010-11-02\" and daydate <= \"2010-11-04\"'); ");
@@ -366,7 +367,7 @@ public class TestAllLoader extends TestC
@Test
public void testDateParitionDir() throws IOException, ParseException {
- server.registerQuery("a = LOAD '" + datePartitionDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionDir.getAbsolutePath())
+ "' using " + allLoaderFuncSpec
+ " as (q:float, p:float, daydate:chararray);");
@@ -399,7 +400,7 @@ public class TestAllLoader extends TestC
@Test
public void testSimpleDir() throws IOException {
- server.registerQuery("a = LOAD '" + simpleDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(simpleDir.getAbsolutePath())
+ "' using " + allLoaderFuncSpec + " as (q:float, p:float);");
readRecordsFromLoader(server, "a", fileRecords * fileTypes.length);
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCombinedLogLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCombinedLogLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCombinedLogLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCombinedLogLoader.java Thu May 30 18:50:07 2013
@@ -82,7 +82,7 @@ public class TestCombinedLogLoader exten
String filename = TestHelper.createTempFile(data, " ");
PigServer pig = new PigServer(ExecType.LOCAL);
filename = filename.replace("\\", "\\\\");
- pig.registerQuery("A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.apachelog.CombinedLogLoader();");
+ pig.registerQuery("A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.apachelog.CombinedLogLoader();");
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCommonLogLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCommonLogLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCommonLogLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCommonLogLoader.java Thu May 30 18:50:07 2013
@@ -73,7 +73,7 @@ public class TestCommonLogLoader extends
String filename = TestHelper.createTempFile(data, " ");
PigServer pig = new PigServer(ExecType.LOCAL);
filename = filename.replace("\\", "\\\\");
- pig.registerQuery("A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.apachelog.CommonLogLoader();");
+ pig.registerQuery("A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.apachelog.CommonLogLoader();");
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestDBStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestDBStorage.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestDBStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestDBStorage.java Thu May 30 18:50:07 2013
@@ -134,6 +134,7 @@ public class TestDBStorage extends TestC
Util.deleteFile(cluster, INPUT_FILE);
pigServer.shutdown();
dbServer.stop();
+ cluster.shutDown();
File[] dbFiles = new File(TMP_DIR).listFiles(new FilenameFilter() {
@Override
@@ -156,7 +157,7 @@ public class TestDBStorage extends TestC
String insertQuery = "insert into ttt (id, name, ratio, dt) values (?,?,?,?)";
pigServer.setBatchOn();
String dbStore = "org.apache.pig.piggybank.storage.DBStorage('" + driver
- + "', '" + url + "', '" + insertQuery + "');";
+ + "', '" + Util.encodeEscape(url) + "', '" + insertQuery + "');";
pigServer.registerQuery("A = LOAD '" + INPUT_FILE
+ "' as (id:int, fruit:chararray, ratio:double, dt : datetime);");
pigServer.registerQuery("STORE A INTO 'dummy' USING " + dbStore);
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarLoader.java Thu May 30 18:50:07 2013
@@ -131,7 +131,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + singlePartitionedFile + "' using " + funcSpecString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + funcSpecString
+ ";");
Iterator<Tuple> result = server.openIterator("a");
@@ -158,7 +158,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + singlePartitionedDir + "' using " + funcSpecString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedDir) + "' using " + funcSpecString
+ ";");
server.registerQuery("b = foreach a generate f1;");
@@ -187,7 +187,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + singlePartitionedFile + "' using " + funcSpecString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + funcSpecString
+ ";");
server.registerQuery("b = foreach a generate f1;");
@@ -216,7 +216,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + yearMonthDayHourPartitionedDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(yearMonthDayHourPartitionedDir.getAbsolutePath())
+ "' using " + funcSpecString + ";");
server.registerQuery("f = FILTER a by year=='2010';");
server.registerQuery("b = foreach f generate f1,f2;");
@@ -246,7 +246,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + yearMonthDayHourPartitionedDir.getAbsolutePath()
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(yearMonthDayHourPartitionedDir.getAbsolutePath())
+ "' using " + funcSpecString + ";");
server.registerQuery("f = FILTER a by year=='2010';");
server.registerQuery("r = foreach f generate year, f2, f3, month, day, hour;");
@@ -277,7 +277,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + datePartitionedDir.getAbsolutePath() + "' using "
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
+ funcSpecString + ";");
server.registerQuery("b = FOREACH a GENERATE f2 as p;");
Iterator<Tuple> result = server.openIterator("b");
@@ -306,7 +306,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + datePartitionedDir.getAbsolutePath() + "' using "
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
+ funcSpecString + ";");
Iterator<Tuple> result = server.openIterator("a");
@@ -331,7 +331,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + datePartitionedDir.getAbsolutePath() + "' using "
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
+ funcSpecString + ";");
Iterator<Tuple> result = server.openIterator("a");
@@ -355,7 +355,7 @@ public class TestHiveColumnarLoader exte
server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
new FuncSpec(funcSpecString));
- server.registerQuery("a = LOAD '" + datePartitionedDir.getAbsolutePath() + "' using "
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
+ funcSpecString + ";");
Iterator<Tuple> result = server.openIterator("a");
Tuple t = null;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarStorage.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestHiveColumnarStorage.java Thu May 30 18:50:07 2013
@@ -81,7 +81,7 @@ public class TestHiveColumnarStorage ext
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("a = LOAD '" + singlePartitionedFile + "' using " + loadString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
+ ";");
//when
@@ -111,7 +111,7 @@ public class TestHiveColumnarStorage ext
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("a = LOAD '" + singlePartitionedFile + "' using " + loadString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
+ ";");
server.registerQuery("b = FOREACH a GENERATE f1, TOTUPLE(f2,f3);");
@@ -148,7 +148,7 @@ public class TestHiveColumnarStorage ext
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("a = LOAD '" + singlePartitionedFile + "' using " + loadString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
+ ";");
server.registerQuery("b = FOREACH a GENERATE f1, TOBAG(f2,f3);");
@@ -185,7 +185,7 @@ public class TestHiveColumnarStorage ext
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("a = LOAD '" + singlePartitionedFile + "' using " + loadString
+ server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
+ ";");
server.registerQuery("b = FOREACH a GENERATE f1, TOMAP(f2,f3);");
@@ -233,6 +233,7 @@ public class TestHiveColumnarStorage ext
LongWritable key = rcFileRecordReader.createKey();
BytesRefArrayWritable value = rcFileRecordReader.createValue();
rcFileRecordReader.next(key, value);
+ rcFileRecordReader.close();
ColumnarStruct struct = readColumnarStruct(value, schema);
return struct;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java Thu May 30 18:50:07 2013
@@ -96,7 +96,7 @@ public class TestIndexedStorage {
input.delete();
Util.createLocalInputFile(input.getAbsolutePath(), inputs);
- pigServer.registerQuery("A = load '" + input.getAbsolutePath() + "' as (a0:int, a1:int);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(input.getAbsolutePath()) + "' as (a0:int, a1:int);");
File output = new File(outputDir, "/" + id);
pigServer.store("A", output.getAbsolutePath(), "org.apache.pig.piggybank.storage.IndexedStorage('\t','0,1')");
@@ -124,7 +124,7 @@ public class TestIndexedStorage {
TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 1, true, 1, 1);
conf.set("mapred.task.id", taskId.toString());
- conf.set("mapred.input.dir", outputDir.getAbsolutePath());
+ conf.set("mapred.input.dir", Util.encodeEscape(outputDir.getAbsolutePath()));
storage.initialize(conf);
Integer key;
@@ -156,7 +156,7 @@ public class TestIndexedStorage {
TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 2, true, 2, 2);
conf.set("mapred.task.id", taskId.toString());
- conf.set("mapred.input.dir", outputDir.getAbsolutePath());
+ conf.set("mapred.input.dir", Util.encodeEscape(outputDir.getAbsolutePath()));
storage.initialize(conf);
TupleFactory tupleFactory = TupleFactory.getInstance();
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLoadFuncHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLoadFuncHelper.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLoadFuncHelper.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLoadFuncHelper.java Thu May 30 18:50:07 2013
@@ -75,7 +75,7 @@ public class TestLoadFuncHelper extends
Path path = helper.determineFirstFile(baseDir.getAbsolutePath());
assertNotNull(path);
- assertEquals(testFile.getAbsolutePath(), path.toUri().toURL().getFile());
+ assertEquals(new Path(testFile.getAbsolutePath()).toUri().toString(), path.toUri().toURL().getFile());
}
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java Thu May 30 18:50:07 2013
@@ -94,6 +94,7 @@ public class TestMultiStorage extends Te
public void tearDown() throws Exception {
new File(INPUT_FILE).delete();
Util.deleteFile(cluster, INPUT_FILE);
+ cluster.shutDown();
}
enum Mode {
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java Thu May 30 18:50:07 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.compress.Gzi
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.test.Util;
public class TestMultiStorageCompression extends TestCase {
@@ -163,12 +164,12 @@ public class TestMultiStorageCompression
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename
+ String query = "A = LOAD '" + Util.encodeEscape(filename)
+ "' USING PigStorage(',') as (a,b,c);";
- String query2 = "STORE A INTO '" + outputPath
+ String query2 = "STORE A INTO '" + Util.encodeEscape(outputPath)
+ "' USING org.apache.pig.piggybank.storage.MultiStorage" + "('"
- + outputPath + "','0', '" + compressionType + "', '\\t');";
+ + Util.encodeEscape(outputPath) + "','0', '" + compressionType + "', '\\t');";
// Run Pig
pig.setBatchOn();
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMyRegExLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMyRegExLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMyRegExLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMyRegExLoader.java Thu May 30 18:50:07 2013
@@ -40,7 +40,7 @@ public class TestMyRegExLoader extends T
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.MyRegExLoader('" + patternString + "');";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.MyRegExLoader('" + patternString + "');";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestPathPartitioner.java Thu May 30 18:50:07 2013
@@ -75,7 +75,7 @@ public class TestPathPartitioner extends
PathPartitioner partitioner = new PathPartitioner();
Map<String, String> map = partitioner
- .getPathPartitionKeyValues(partition3.getAbsolutePath());
+ .getPathPartitionKeyValues(partition3.getAbsolutePath().replaceAll("\\\\", "/"));
String[] keys = map.keySet().toArray(new String[] {});
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java Thu May 30 18:50:07 2013
@@ -66,7 +66,7 @@ public class TestRegExLoader extends Tes
String filename = TestHelper.createTempFile(data, "");
ArrayList<DataByteArray[]> expected = TestHelper.getExpected(data, pattern);
- pigServer.registerQuery("A = LOAD 'file:" + Util.encodeEscape(filename) +
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(filename) +
"' USING " + DummyRegExLoader.class.getName() + "() AS (key, val);");
Iterator<?> it = pigServer.openIterator("A");
int tupleCount = 0;
@@ -92,7 +92,7 @@ public class TestRegExLoader extends Tes
dataE.add(new String[] { "3,three;iii" });
ArrayList<DataByteArray[]> expected = TestHelper.getExpected(dataE, pattern2);
- pigServer.registerQuery("A = LOAD 'file:" + Util.encodeEscape(filename) +
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(filename) +
"' USING " + DummyRegExLoader2.class.getName() + "() AS (key, val);");
Iterator<?> it = pigServer.openIterator("A");
int tupleCount = 0;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestSequenceFileLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestSequenceFileLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestSequenceFileLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestSequenceFileLoader.java Thu May 30 18:50:07 2013
@@ -81,7 +81,7 @@ import org.apache.pig.test.Util;
@Test
public void testReadsNocast() throws IOException {
- pigServer.registerQuery("A = LOAD 'file:" + Util.encodeEscape(tmpFileName) +
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(tmpFileName) +
"' USING org.apache.pig.piggybank.storage.SequenceFileLoader() AS (key, val);");
Iterator<?> it = pigServer.openIterator("A");
int tupleCount = 0;
@@ -101,7 +101,7 @@ import org.apache.pig.test.Util;
@Test
public void testReadsStringCast() throws IOException {
- pigServer.registerQuery("A = LOAD 'file:" + Util.encodeEscape(tmpFileName) +
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(tmpFileName) +
"' USING org.apache.pig.piggybank.storage.SequenceFileLoader() AS (key:long, val);");
Iterator<?> it = pigServer.openIterator("A");
int tupleCount = 0;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java Thu May 30 18:50:07 2013
@@ -28,6 +28,7 @@ import junit.framework.TestCase;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
+import org.apache.pig.test.Util;
public class TestXMLLoader extends TestCase {
private static String patternString = "(\\d+)!+(\\w+)~+(\\w+)";
@@ -95,7 +96,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('invalid') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('invalid') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -118,7 +119,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('property') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('property') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -150,7 +151,7 @@ public class TestXMLLoader extends TestC
try
{
PigServer pigServer = new PigServer (ExecType.LOCAL);
- String loadQuery = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('property') as (doc:chararray);";
+ String loadQuery = "A = LOAD '" + Util.encodeEscape(filename) + "' USING org.apache.pig.piggybank.storage.XMLLoader('property') as (doc:chararray);";
pigServer.registerQuery(loadQuery);
Iterator<Tuple> it = pigServer.openIterator("A");
int tupleCount = 0;
@@ -192,7 +193,7 @@ public class TestXMLLoader extends TestC
{
PigServer pigServer = new PigServer (ExecType.LOCAL);
- String loadQuery = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('property') as (doc:chararray);";
+ String loadQuery = "A = LOAD '" + Util.encodeEscape(filename) + "' USING org.apache.pig.piggybank.storage.XMLLoader('property') as (doc:chararray);";
pigServer.registerQuery(loadQuery);
Iterator<Tuple> it = pigServer.openIterator("A");
@@ -224,7 +225,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('name') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('name') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -248,7 +249,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('ignoreProperty') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('ignoreProperty') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -281,7 +282,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('</ignoreProperty>') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('</ignoreProperty>') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -307,7 +308,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('</ignoreProperty>') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('</ignoreProperty>') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -330,7 +331,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('event') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('event') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -353,7 +354,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('event') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('event') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
int tupleCount = 0;
@@ -375,7 +376,7 @@ public class TestXMLLoader extends TestC
PigServer pig = new PigServer(LOCAL);
filename = filename.replace("\\", "\\\\");
patternString = patternString.replace("\\", "\\\\");
- String query = "A = LOAD 'file:" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('event') as (doc:chararray);";
+ String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('event') as (doc:chararray);";
pig.registerQuery(query);
Iterator<?> it = pig.openIterator("A");
while (it.hasNext()) {
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Thu May 30 18:50:07 2013
@@ -75,12 +75,12 @@ public class TestAvroStorage {
private static String getInputFile(String file) {
String locations[] = LoadFunc.getPathStrings(file);
if (locations.length == 1)
- return "file://" + System.getProperty("user.dir") + "/" + basedir
+ return System.getProperty("user.dir") + "/" + basedir
+ file;
else {
ArrayList<String> pathStrings = new ArrayList<String>();
for (int index = 0; index < locations.length; index++) {
- String f = "file://" + System.getProperty("user.dir") + "/"
+ String f = System.getProperty("user.dir") + "/"
+ basedir + locations[index].trim();
pathStrings.add(f);
}
@@ -203,7 +203,7 @@ public class TestAvroStorage {
String expected = testRecursiveRecordInMap;
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInMap +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInMap) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
@@ -221,7 +221,7 @@ public class TestAvroStorage {
String expected = testRecursiveRecordInArray;
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInArray +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInArray) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
@@ -239,7 +239,7 @@ public class TestAvroStorage {
String expected = testRecursiveRecordInUnion;
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
@@ -257,12 +257,12 @@ public class TestAvroStorage {
String expected = testRecursiveRecordInRecord;
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInRecord +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInRecord) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'no_schema_check'," +
- " 'schema', '" + recursiveRecordInRecord + "' );"
+ " 'schema', '" + Util.encodeEscape(recursiveRecordInRecord) + "' );"
};
testAvroStorage(queries);
verifyResults(output, expected);
@@ -276,12 +276,12 @@ public class TestAvroStorage {
String expected = testRecursiveRecordInUnion;
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'no_schema_check'," +
- " 'same', '" + testRecursiveRecordInUnion + "' );"
+ " 'same', '" + Util.encodeEscape(testRecursiveRecordInUnion) + "' );"
};
testAvroStorage(queries);
verifyResults(output, expected);
@@ -305,7 +305,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecursiveRecordReference1.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" first = FOREACH in GENERATE $0 AS value;",
" filtered = FILTER first BY value is not null;",
@@ -335,7 +335,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecursiveRecordReference2.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" second = FOREACH in GENERATE $1.$0 AS value;",
" filtered = FILTER second BY value is not null;",
@@ -364,7 +364,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecursiveRecordReference3.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" third = FOREACH in GENERATE $1.$1.$0 AS value;",
" filtered = FILTER third BY value is not null;",
@@ -384,7 +384,7 @@ public class TestAvroStorage {
String output= outbasedir + "testRecursiveRecordWithNoAvroSchema";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
@@ -402,7 +402,7 @@ public class TestAvroStorage {
String output= outbasedir + "testRecursiveWithSchemaCheck";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
@@ -424,12 +424,12 @@ public class TestAvroStorage {
String output= outbasedir + "testRecursiveWithSchemaFile";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'no_schema_check'," +
- " 'schema_file', '" + testRecursiveRecordInUnionSchema + "' );"
+ " 'schema_file', '" + Util.encodeEscape(testRecursiveRecordInUnionSchema) + "' );"
};
try {
testAvroStorage(queries);
@@ -447,12 +447,12 @@ public class TestAvroStorage {
String output= outbasedir + "testRecursiveWithData";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testRecursiveRecordInUnion +
+ " in = LOAD '" + Util.encodeEscape(testRecursiveRecordInUnion) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'no_schema_check'," +
- " 'data', '" + testRecursiveRecordInUnion + "' );"
+ " 'data', '" + Util.encodeEscape(testRecursiveRecordInUnion) + "' );"
};
try {
testAvroStorage(queries);
@@ -470,7 +470,7 @@ public class TestAvroStorage {
String output= outbasedir + "testGenericUnion";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testGenericUnionFile +
+ " in = LOAD '" + Util.encodeEscape(testGenericUnionFile) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
@@ -515,7 +515,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testMultipleSchemas1.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testMultipleSchemas1File +
+ " in = LOAD '" + Util.encodeEscape(testMultipleSchemas1File) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ('multiple_schemas');",
" s = FOREACH in GENERATE StringConcat($0);",
" o = ORDER s BY $0;",
@@ -567,7 +567,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testMultipleSchemas2.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testMultipleSchemas2File +
+ " in = LOAD '" + Util.encodeEscape(testMultipleSchemas2File) +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ('multiple_schemas');",
" f = FOREACH in GENERATE ($0 is not null ? (chararray)$0 : '')," +
" ($1 is not null ? (chararray)$1 : '')," +
@@ -626,7 +626,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testDir.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testDir1 + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testDir1) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -641,7 +641,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testDir.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testDir1AllFiles + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testDir1AllFiles) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -656,7 +656,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_test_dir_1.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testDir1Files123 + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testDir1Files123) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -671,7 +671,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_test_dir_1.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testDir1Files321 + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testDir1Files321) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -686,7 +686,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_test_dir_1_2.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testDir12AllFiles + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testDir12AllFiles) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -701,7 +701,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_test_dir_1_2.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testDir21AllFiles + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testDir21AllFiles) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -715,7 +715,7 @@ public class TestAvroStorage {
String output = outbasedir + "testGlob6";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testNoMatchedFiles + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testNoMatchedFiles) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -737,7 +737,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_test_dir_1.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testCommaSeparated1 + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testCommaSeparated1) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -752,7 +752,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_test_dir_1_2.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testCommaSeparated2 + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testCommaSeparated2) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
};
@@ -768,7 +768,7 @@ public class TestAvroStorage {
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testArrayFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
};
testAvroStorage( queries);
@@ -781,7 +781,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testArrayWithSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testArrayFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( " +
" 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
@@ -796,10 +796,10 @@ public class TestAvroStorage {
String expected = basedir + "expected_testArrayWithSchemaURI.avro"; // doubles (not floats) stored
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testArrayFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( " +
- " 'schema_uri', '" + testArraySchema + "' );"
+ " 'schema_uri', '" + Util.encodeEscape(testArraySchema) + "' );"
};
testAvroStorage( queries);
verifyResults(output, expected);
@@ -811,7 +811,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testArrayWithSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testArrayFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( " +
" '{\"nullable\": false }' );"
@@ -826,10 +826,10 @@ public class TestAvroStorage {
String expected = basedir + "expected_testArrayWithSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testArrayFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( " +
- " 'same', '" + testArrayFile + "' );"
+ " 'same', '" + Util.encodeEscape(testArrayFile) + "' );"
};
testAvroStorage(queries);
verifyResults(output, expected);
@@ -849,7 +849,7 @@ public class TestAvroStorage {
PigServer pigServer = new PigServer(ExecType.LOCAL, properties);
pigServer.setBatchOn();
String [] queries = {
- " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testArrayFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
};
for (String query: queries){
@@ -869,7 +869,7 @@ public class TestAvroStorage {
deleteDirectory(new File(output1));
deleteDirectory(new File(output2));
String [] queries = {
- " avro = LOAD '" + testRecordFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " avro = LOAD '" + Util.encodeEscape(testRecordFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" groups = GROUP avro BY member_id;",
" sc = FOREACH groups GENERATE group AS key, COUNT(avro) AS cnt;",
" STORE sc INTO '" + output1 + "' " +
@@ -900,7 +900,7 @@ public class TestAvroStorage {
deleteDirectory(new File(output1));
deleteDirectory(new File(output2));
String [] queries = {
- " avro = LOAD '" + testTextFile + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
+ " avro = LOAD '" + Util.encodeEscape(testTextFile) + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
" groups = GROUP avro BY member_id;",
" sc = FOREACH groups GENERATE group AS key, COUNT(avro) AS cnt;",
" STORE sc INTO '" + output1 + "' " +
@@ -928,12 +928,12 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecordWithFieldSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " avro = LOAD '" + testRecordFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " avro = LOAD '" + Util.encodeEscape(testRecordFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" avro1 = FILTER avro BY member_id > 1211;",
" avro2 = FOREACH avro1 GENERATE member_id, browser_id, tracking_time, act_content ;",
" STORE avro2 INTO '" + output + "' " +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
- "'{\"data\": \"" + testRecordFile + "\" ," +
+ "'{\"data\": \"" + Util.encodeEscape(testRecordFile) + "\" ," +
" \"field0\": \"int\", " +
" \"field1\": \"def:browser_id\", " +
" \"field3\": \"def:act_content\" " +
@@ -950,12 +950,12 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecordWithFieldSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " avro = LOAD '" + testTextFile + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
+ " avro = LOAD '" + Util.encodeEscape(testTextFile) + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
" avro1 = FILTER avro BY member_id > 1211;",
" avro2 = FOREACH avro1 GENERATE member_id, browser_id, tracking_time, act_content ;",
" STORE avro2 INTO '" + output + "' " +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
- "'{\"data\": \"" + testRecordFile + "\" ," +
+ "'{\"data\": \"" + Util.encodeEscape(testRecordFile) + "\" ," +
" \"field0\": \"int\", " +
" \"field1\": \"def:browser_id\", " +
" \"field3\": \"def:act_content\" " +
@@ -972,12 +972,12 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecordWithFieldSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " avro = LOAD '" + testTextFile + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
+ " avro = LOAD '" + Util.encodeEscape(testTextFile) + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
" avro1 = FILTER avro BY member_id > 1211;",
" avro2 = FOREACH avro1 GENERATE member_id, browser_id, tracking_time, act_content ;",
" STORE avro2 INTO '" + output + "' " +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
- "'{\"schema_file\": \"" + testRecordSchema + "\" ," +
+ "'{\"schema_file\": \"" + Util.encodeEscape(testRecordSchema) + "\" ," +
" \"field0\": \"int\", " +
" \"field1\": \"def:browser_id\", " +
" \"field3\": \"def:act_content\" " +
@@ -993,7 +993,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testSingleFieldTuples.avro";
deleteDirectory(new File(output));
String [] queries = {
- " messages = LOAD '" + testSingleTupleBagFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " messages = LOAD '" + Util.encodeEscape(testSingleTupleBagFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" a = foreach (group messages by user_id) { sorted = order messages by message_id DESC; GENERATE group AS user_id, sorted AS messages; };",
" STORE a INTO '" + output + "' " +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
@@ -1008,12 +1008,12 @@ public class TestAvroStorage {
String expected = basedir + "expected_testFileWithNoExtension.avro";
deleteDirectory(new File(output));
String [] queries = {
- " avro = LOAD '" + testNoExtensionFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " avro = LOAD '" + Util.encodeEscape(testNoExtensionFile) + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" avro1 = FILTER avro BY member_id > 1211;",
" avro2 = FOREACH avro1 GENERATE member_id, browser_id, tracking_time, act_content ;",
" STORE avro2 INTO '" + output + "' " +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
- "'{\"data\": \"" + testNoExtensionFile + "\" ," +
+ "'{\"data\": \"" + Util.encodeEscape(testNoExtensionFile) + "\" ," +
" \"field0\": \"int\", " +
" \"field1\": \"def:browser_id\", " +
" \"field3\": \"def:act_content\" " +
@@ -1031,12 +1031,12 @@ public class TestAvroStorage {
String expected = basedir + "expected_testRecordWithFieldSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
- " avro = LOAD '" + testTextFile + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
+ " avro = LOAD '" + Util.encodeEscape(testTextFile) + "' AS (member_id:int, browser_id:chararray, tracking_time:long, act_content:bag{inner:tuple(key:chararray, value:chararray)});",
" avro1 = FILTER avro BY member_id > 1211;",
" avro2 = FOREACH avro1 GENERATE member_id, browser_id, tracking_time, act_content ;",
" STORE avro2 INTO '" + output + "' " +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
- "'schema_file', '" + testRecordSchema + "'," +
+ "'schema_file', '" + Util.encodeEscape(testRecordSchema) + "'," +
"'field0','int'," +
"'field1','def:browser_id'," +
"'field3','def:act_content'" +
@@ -1052,7 +1052,7 @@ public class TestAvroStorage {
String output = outbasedir + "testCorruptedFile1";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testCorruptedFile + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " in = LOAD '" + Util.encodeEscape(testCorruptedFile) + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
};
// Job is expected to fail for bad files.
@@ -1067,7 +1067,7 @@ public class TestAvroStorage {
String expected = basedir + "expected_testCorruptedFile.avro";
deleteDirectory(new File(output));
String [] queries = {
- " in = LOAD '" + testCorruptedFile + "'" +
+ " in = LOAD '" + Util.encodeEscape(testCorruptedFile) + "'" +
" USING org.apache.pig.piggybank.storage.avro.AvroStorage ('ignore_bad_files');",
" STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
};
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java?rev=1487970&r1=1487969&r2=1487970&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java Thu May 30 18:50:07 2013
@@ -94,7 +94,7 @@ public class TestAvroStorageUtils {
@Test
public void testGetAllSubDirs() throws IOException {
- final String basedir = "file://" + System.getProperty("user.dir");
+ final String basedir = System.getProperty("user.dir");
final String tempdir = Long.toString(System.currentTimeMillis());
final String nonexistentpath = basedir + "/" + tempdir + "/this_path_does_not_exist";