You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/10/31 22:28:02 UTC
svn commit: r1404361 - in /pig/branches/branch-0.10: CHANGES.txt
test/org/apache/pig/test/TestBuiltin.java
test/org/apache/pig/test/TestJoin.java
test/org/apache/pig/test/TestPigScriptParser.java
test/org/apache/pig/test/TestUnion.java
Author: daijy
Date: Wed Oct 31 21:28:01 2012
New Revision: 1404361
URL: http://svn.apache.org/viewvc?rev=1404361&view=rev
Log:
PIG-2797: Tests should not create their own file URIs through string concatenation, should use Util.generateURI instead (jgordon via daijy)
Modified:
pig/branches/branch-0.10/CHANGES.txt
pig/branches/branch-0.10/test/org/apache/pig/test/TestBuiltin.java
pig/branches/branch-0.10/test/org/apache/pig/test/TestJoin.java
pig/branches/branch-0.10/test/org/apache/pig/test/TestPigScriptParser.java
pig/branches/branch-0.10/test/org/apache/pig/test/TestUnion.java
Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1404361&r1=1404360&r2=1404361&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Wed Oct 31 21:28:01 2012
@@ -36,6 +36,9 @@ PIG-2727: PigStorage Source tagging does
BUG FIXES
+PIG-2797: Tests should not create their own file URIs through string concatenation, should use Util.generateURI
+instead (jgordon via daijy)
+
PIG-2796: Local temporary paths are not always valid HDFS path names (jgordon via daijy)
PIG-2795: Fix test cases that generate pig scripts with "load " + pathStr to encode "\" in the path (jgordon via daijy)
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestBuiltin.java?rev=1404361&r1=1404360&r2=1404361&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestBuiltin.java Wed Oct 31 21:28:01 2012
@@ -162,7 +162,7 @@ public class TestBuiltin {
@Before
public void setUp() throws Exception {
- pigServer = new PigServer(ExecType.LOCAL, new Properties());
+ pigServer = new PigServer(ExecType.LOCAL, cluster.getProperties());
pigServer.setValidateEachStatement(true);
// First set up data structs for "base" SUM, MIN and MAX and AVG.
// The allowed input and expected output data structs for
@@ -867,7 +867,7 @@ public class TestBuiltin {
public void testCount_ValidNumberOfArguments_WithoutInputSchema_One() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "';");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
assertValidCount();
@@ -882,7 +882,7 @@ public class TestBuiltin {
public void testCount_ValidNumberOfArguments_WithoutInputSchema_Two() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "';");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A);");
assertValidCount();
@@ -905,7 +905,7 @@ public class TestBuiltin {
File inputFile = Util.createInputFile("tmp", inputFileName, inputData);
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "';");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A);");
assertValidCount();
@@ -928,7 +928,7 @@ public class TestBuiltin {
String inputFileName = file.getAbsolutePath();
try {
- pigServer.registerQuery("A = load '" + inputFileName + "';");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFileName) + "';");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
assertValidCount();
@@ -944,7 +944,7 @@ public class TestBuiltin {
public void testCount_ValidNumberOfArguments_WithInputSchema_One() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' as (data:chararray);");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
assertValidCount();
@@ -959,7 +959,7 @@ public class TestBuiltin {
public void testCount_ValidNumberOfArguments_WithInputSchema_Two() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' as (data:chararray);");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A);");
assertValidCount();
@@ -995,7 +995,7 @@ public class TestBuiltin {
public void testCount_InvalidNumberOfArguments_WithoutInputSchema() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "';");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "';");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
pigServer.openIterator("C");
@@ -1011,7 +1011,7 @@ public class TestBuiltin {
public void testCount_InvalidNumberOfArguments_WithInputSchema() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' as (data:chararray);");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
pigServer.openIterator("C");
@@ -1027,7 +1027,7 @@ public class TestBuiltin {
public void testCount_InvalidArgumentType() throws Exception {
File inputFile = createCountInputFile();
try {
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (data:chararray);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' as (data:chararray);");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT('data');");
pigServer.openIterator("C");
@@ -2079,7 +2079,7 @@ public class TestBuiltin {
File inputFile = Util.createInputFile("tmp", inputFileName, inputData);
// test typed data
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' AS value:chararray;");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' AS value:chararray;");
pigServer.registerQuery("B = foreach A generate STRSPLIT(value, ' ') AS values;");
pigServer.registerQuery("C = foreach B generate values, SIZE(values) as cnt;");
@@ -2226,7 +2226,7 @@ public class TestBuiltin {
File inputFile = Util.createInputFile("tmp", "testStrUDFsIn.txt", new String[] {inputStr});
// test typed data
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (name: chararray);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' as (name: chararray);");
pigServer.registerQuery("B = foreach A generate SUBSTRING(name, 0, 3), " +
"INDEXOF(name, 'a'), INDEXOF(name, 'a', 3), LAST_INDEX_OF(name, 'a'), REPLACE(name, 'a', 'b'), " +
"STRSPLIT(name), STRSPLIT(name, ' '), STRSPLIT(name, ' ', 0), TRIM(name);");
@@ -2248,7 +2248,7 @@ public class TestBuiltin {
assertEquals("amy smith", t.get(8));
// test untyped data
- pigServer.registerQuery("A = load '" + inputFile.getAbsolutePath() + "' as (name);");
+ pigServer.registerQuery("A = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "' as (name);");
pigServer.registerQuery("B = foreach A generate SUBSTRING(name, 0, 3), " +
"LAST_INDEX_OF(name, 'a'), REPLACE(name, 'a', 'b'), TRIM(name);");
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestJoin.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestJoin.java?rev=1404361&r1=1404360&r2=1404361&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestJoin.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestJoin.java Wed Oct 31 21:28:01 2012
@@ -92,7 +92,7 @@ public class TestJoin extends TestCase {
fileName = fileNameHint;
} else if (execType == ExecType.LOCAL) {
File f = Util.createInputFile("test", fileNameHint, data);
- fileName = "file://" + f.getAbsolutePath();
+ fileName = Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), pigServer.getPigContext());
}
return fileName;
}
@@ -173,8 +173,8 @@ public class TestJoin extends TestCase {
Tuple expectedResult = (Tuple)Util.getPigConstant("('hello',1,'hello','world')");
// with schema
- String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
- "b = load '"+ secondInput +"' as (n:chararray, m:chararray); " +
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray, a:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray, m:chararray); " +
"c = join a by $0, b by $0;";
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("c");
@@ -183,8 +183,8 @@ public class TestJoin extends TestCase {
assertEquals(false, it.hasNext());
// without schema
- script = "a = load '"+ firstInput + "'; " +
- "b = load '" + secondInput + "'; " +
+ script = "a = load '"+ Util.encodeEscape(firstInput) + "'; " +
+ "b = load '" + Util.encodeEscape(secondInput) + "'; " +
"c = join a by $0, b by $0;";
Util.registerMultiLineQuery(pigServer, script);
it = pigServer.openIterator("c");
@@ -216,8 +216,8 @@ public class TestJoin extends TestCase {
Tuple expectedResult = (Tuple)Util.getPigConstant("(1,2,1,'hello',1,2,1,'hello')");
// with schema
- String script = "a = load '"+ firstInput +"' as (i:int, j:int); " +
- "b = load '"+ secondInput +"' as (k:int, l:chararray); " +
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (i:int, j:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (k:int, l:chararray); " +
"c = join a by $0, b by $0;" +
"d = foreach c generate i,j,k,l,a::i as ai,a::j as aj,b::k as bk,b::l as bl;";
Util.registerMultiLineQuery(pigServer, script);
@@ -227,8 +227,8 @@ public class TestJoin extends TestCase {
assertEquals(false, it.hasNext());
// schema with duplicates
- script = "a = load '"+ firstInput +"' as (i:int, j:int); " +
- "b = load '"+ secondInput +"' as (i:int, l:chararray); " +
+ script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (i:int, j:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (i:int, l:chararray); " +
"c = join a by $0, b by $0;" +
"d = foreach c generate i,j,l,a::i as ai,a::j as aj,b::i as bi,b::l as bl;";
boolean exceptionThrown = false;
@@ -243,8 +243,8 @@ public class TestJoin extends TestCase {
assertEquals(true, exceptionThrown);
// schema with duplicates with resolution
- script = "a = load '"+ firstInput +"' as (i:int, j:int); " +
- "b = load '"+ secondInput +"' as (i:int, l:chararray); " +
+ script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (i:int, j:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (i:int, l:chararray); " +
"c = join a by $0, b by $0;" +
"d = foreach c generate a::i as ai1,j,b::i as bi1,l,a::i as ai2,a::j as aj2,b::i as bi3,b::l as bl3;";
Util.registerMultiLineQuery(pigServer, script);
@@ -283,11 +283,11 @@ public class TestJoin extends TestCase {
}
// with schema
- String script = "a = load '"+ firstInput +"' ; " +
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' ; " +
//re-using alias a for new operator below, doing this intentionally
// because such use case has been seen
"a = foreach a generate $0 as i, $1 as j ;" +
- "b = load '"+ secondInput +"' as (k, l); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (k, l); " +
"c = join a by $0, b by $0;" +
"d = foreach c generate i,j,k,l,a::i as ai,a::j as aj,b::k as bk,b::l as bl;";
Util.registerMultiLineQuery(pigServer, script);
@@ -329,8 +329,8 @@ public class TestJoin extends TestCase {
// with and without optional outer
for(int i = 0; i < 2; i++) {
//with schema
- String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
- "b = load '"+ secondInput +"' as (n:chararray, m:chararray); ";
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray, a:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray, m:chararray); ";
if(i == 0) {
script += "c = join a by $0 left outer, b by $0;" ;
} else {
@@ -351,8 +351,8 @@ public class TestJoin extends TestCase {
assertEquals(expectedResults.size(), counter);
// without schema
- script = "a = load '"+ firstInput +"'; " +
- "b = load '"+ secondInput +"'; ";
+ script = "a = load '"+ Util.encodeEscape(firstInput) +"'; " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"'; ";
if(i == 0) {
script += "c = join a by $0 left outer, b by $0;" ;
} else {
@@ -398,8 +398,8 @@ public class TestJoin extends TestCase {
// with and without optional outer
for(int i = 0; i < 2; i++) {
// with schema
- String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
- "b = load '"+ secondInput +"' as (n:chararray, m:chararray); ";
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray, a:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray, m:chararray); ";
if(i == 0) {
script += "c = join a by $0 right outer, b by $0;" ;
} else {
@@ -420,8 +420,8 @@ public class TestJoin extends TestCase {
assertEquals(expectedResults.size(), counter);
// without schema
- script = "a = load '"+ firstInput +"'; " +
- "b = load '"+ secondInput +"'; " ;
+ script = "a = load '"+ Util.encodeEscape(firstInput) +"'; " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"'; " ;
if(i == 0) {
script += "c = join a by $0 right outer, b by $0;" ;
} else {
@@ -469,8 +469,8 @@ public class TestJoin extends TestCase {
// with and without optional outer
for(int i = 0; i < 2; i++) {
// with schema
- String script = "a = load '"+ firstInput +"' as (n:chararray, a:int); " +
- "b = load '"+ secondInput +"' as (n:chararray, m:chararray); ";
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (n:chararray, a:int); " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (n:chararray, m:chararray); ";
if(i == 0) {
script += "c = join a by $0 full outer, b by $0;" ;
} else {
@@ -491,8 +491,8 @@ public class TestJoin extends TestCase {
assertEquals(expectedResults.size(), counter);
// without schema
- script = "a = load '"+ firstInput +"'; " +
- "b = load '"+ secondInput +"'; " ;
+ script = "a = load '"+ Util.encodeEscape(firstInput) +"'; " +
+ "b = load '"+ Util.encodeEscape(secondInput) +"'; " ;
if(i == 0) {
script += "c = join a by $0 full outer, b by $0;" ;
} else {
@@ -577,8 +577,8 @@ public class TestJoin extends TestCase {
String firstInput = createInputFile(execType, "a.txt", input1);
String secondInput = createInputFile(execType, "b.txt", input2);
- String script = "a = load '"+ firstInput +"' as (a:tuple(a1:int, a2:chararray));" +
- "b = load '"+ secondInput +"' as (b:tuple(b1:int, b2:chararray));" +
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (a:tuple(a1:int, a2:chararray));" +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (b:tuple(b1:int, b2:chararray));" +
"c = join a by a.a1, b by b.b1;";
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("c");
@@ -612,8 +612,8 @@ public class TestJoin extends TestCase {
String firstInput = createInputFile(execType, "a.txt", input1);
String secondInput = createInputFile(execType, "b.txt", input2);
- String script = "a = load '"+ firstInput +"' as (a1:int, a2:chararray);" +
- "b = load '"+ secondInput +"' as (b1:int, b2:chararray);" +
+ String script = "a = load '"+ Util.encodeEscape(firstInput) +"' as (a1:int, a2:chararray);" +
+ "b = load '"+ Util.encodeEscape(secondInput) +"' as (b1:int, b2:chararray);" +
"c = join a by (a1, a2), b by (b1, b2);";
Util.registerMultiLineQuery(pigServer, script);
Iterator<Tuple> it = pigServer.openIterator("c");
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestPigScriptParser.java?rev=1404361&r1=1404360&r2=1404361&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestPigScriptParser.java Wed Oct 31 21:28:01 2012
@@ -100,6 +100,7 @@ public class TestPigScriptParser extends
@Test
public void testDefineUDF() throws Exception {
+ PigServer ps = new PigServer(ExecType.LOCAL);
String inputData[] = {
"dshfdskfwww.xyz.com/sportsjoadfjdslpdshfdskfwww.xyz.com/sportsjoadfjdsl" ,
"kas;dka;sd" ,
@@ -108,15 +109,15 @@ public class TestPigScriptParser extends
"wwwJxyzMcom/sports"
};
File f = Util.createFile(inputData);
+
String[] queryLines = new String[] {
// the reason we have 4 backslashes below is we really want to put two backslashes but
// since this is to be represented in a Java String, we escape each backslash with one more
// backslash - hence 4. In a pig script in a file, this would be
// www\\.xyz\\.com
"define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');" ,
- "A = load 'file://" + f.getAbsolutePath() + "' using PigStorage() as (source : chararray);" ,
+ "A = load '" + Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), ps.getPigContext()) + "' using PigStorage() as (source : chararray);" ,
"B = foreach A generate minelogs(source) as sportslogs;" };
- PigServer ps = new PigServer(ExecType.LOCAL);
for (String line : queryLines) {
ps.registerQuery(line);
}
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestUnion.java?rev=1404361&r1=1404360&r2=1404361&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestUnion.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestUnion.java Wed Oct 31 21:28:01 2012
@@ -95,7 +95,7 @@ public class TestUnion extends junit.fra
POLoad ld1 = GenPhyOp.topLoadOp();
String curDir = System.getProperty("user.dir");
String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
- FileSpec fSpec = new FileSpec("file:"+ inpDir +"passwd", new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
+ FileSpec fSpec = new FileSpec(Util.generateURI(Util.encodeEscape(inpDir + "passwd"), pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
ld1.setLFile(fSpec);
POLoad ld2 = GenPhyOp.topLoadOp();