You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/10/31 22:22:41 UTC
svn commit: r1404358 - in /pig/trunk: ./ test/org/apache/pig/test/
Author: daijy
Date: Wed Oct 31 21:22:41 2012
New Revision: 1404358
URL: http://svn.apache.org/viewvc?rev=1404358&view=rev
Log:
PIG-2796: Local temporary paths are not always valid HDFS path names (jgordon via daijy)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/org/apache/pig/test/TestBZip.java
pig/trunk/test/org/apache/pig/test/TestCombiner.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
pig/trunk/test/org/apache/pig/test/TestFinish.java
pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java
pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
pig/trunk/test/org/apache/pig/test/TestPigContext.java
pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
pig/trunk/test/org/apache/pig/test/TestTypedMap.java
pig/trunk/test/org/apache/pig/test/TestUDFContext.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Oct 31 21:22:41 2012
@@ -525,6 +525,8 @@ BUG FIXES
PIG-2795: Fix test cases that generate pig scripts with "load " + pathStr to encode "\" in the path (jgordon via daijy)
+PIG-2796: Local temporary paths are not always valid HDFS path names (jgordon via daijy)
+
Release 0.10.0
INCOMPATIBLE CHANGES
Modified: pig/trunk/test/org/apache/pig/test/TestBZip.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBZip.java Wed Oct 31 21:22:41 2012
@@ -74,8 +74,8 @@ public class TestBZip {
in.deleteOnExit();
File out = File.createTempFile("junit", ".bz2");
- out.deleteOnExit();
out.delete();
+ String clusterOutput = Util.removeColon(out.getAbsolutePath());
CBZip2OutputStream cos =
new CBZip2OutputStream(new FileOutputStream(in));
@@ -88,13 +88,13 @@ public class TestBZip {
cos.close();
pig.registerQuery("AA = load '"
- + Util.generateURI(in.getAbsolutePath(), pig.getPigContext())
+ + Util.generateURI(Util.encodeEscape(in.getAbsolutePath()), pig.getPigContext())
+ "';");
pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
- pig.registerQuery("store A into '" + out.getAbsolutePath() + "';");
+ pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "';");
FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
pig.getPigContext().getProperties()));
- FSDataInputStream is = fs.open(new Path(out.getAbsolutePath() +
+ FSDataInputStream is = fs.open(new Path(clusterOutput +
"/part-r-00000.bz2"));
CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
@@ -103,7 +103,7 @@ public class TestBZip {
assertEquals(100, cis.read(new byte[100]));
cis.close();
- pig.registerQuery("B = load '" + out.getAbsolutePath() + "';");
+ pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
Iterator<Tuple> i = pig.openIterator("B");
HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
@@ -119,7 +119,7 @@ public class TestBZip {
}
in.delete();
- out.delete();
+ Util.deleteFile(cluster, clusterOutput);
}
/**
@@ -133,8 +133,8 @@ public class TestBZip {
in.deleteOnExit();
File out = File.createTempFile("junit", ".bz2");
- out.deleteOnExit();
out.delete();
+ String clusterOutput = Util.removeColon(out.getAbsolutePath());
CBZip2OutputStream cos =
new CBZip2OutputStream(new FileOutputStream(in));
@@ -150,10 +150,10 @@ public class TestBZip {
+ Util.generateURI(in.getAbsolutePath(), pig.getPigContext())
+ "';");
pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
- pig.registerQuery("store A into '" + out.getAbsolutePath() + "/';");
+ pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "/';");
FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
pig.getPigContext().getProperties()));
- FSDataInputStream is = fs.open(new Path(out.getAbsolutePath() +
+ FSDataInputStream is = fs.open(new Path(clusterOutput +
"/part-r-00000.bz2"));
CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
@@ -162,7 +162,7 @@ public class TestBZip {
assertEquals(100, cis.read(new byte[100]));
cis.close();
- pig.registerQuery("B = load '" + out.getAbsolutePath() + "';");
+ pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
Iterator<Tuple> i = pig.openIterator("B");
HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
@@ -247,6 +247,8 @@ public class TestBZip {
File in = File.createTempFile("junit", ".bz2");
String compressedInputFileName = in.getAbsolutePath();
in.deleteOnExit();
+ String clusterCompressedFilePath = Util.removeColon(compressedInputFileName);
+
String unCompressedInputFileName = "testRecordDelims-uncomp.txt";
Util.createInputFile(cluster, unCompressedInputFileName, inputData);
@@ -262,7 +264,7 @@ public class TestBZip {
cos.close();
Util.copyFromLocalToCluster(cluster, compressedInputFileName,
- compressedInputFileName);
+ clusterCompressedFilePath);
// pig script to read uncompressed input
String script = "a = load '" + unCompressedInputFileName +"';";
@@ -272,7 +274,7 @@ public class TestBZip {
Iterator<Tuple> it1 = pig.openIterator("a");
// pig script to read compressed input
- script = "a = load '" + compressedInputFileName +"';";
+ script = "a = load '" + Util.encodeEscape(clusterCompressedFilePath) +"';";
pig.registerQuery(script);
Iterator<Tuple> it2 = pig.openIterator("a");
@@ -287,6 +289,7 @@ public class TestBZip {
} finally {
in.delete();
Util.deleteFile(cluster, unCompressedInputFileName);
+ Util.deleteFile(cluster, clusterCompressedFilePath);
}
}
@@ -303,8 +306,8 @@ public class TestBZip {
in.deleteOnExit();
File out = File.createTempFile("junit", ".bz2");
- out.deleteOnExit();
out.delete();
+ String clusterOutputFilePath = Util.removeColon(out.getAbsolutePath());
FileOutputStream fos = new FileOutputStream(in);
fos.write("55\n".getBytes());
@@ -312,13 +315,13 @@ public class TestBZip {
System.out.println(in.getAbsolutePath());
pig.registerQuery("AA = load '"
- + Util.generateURI(in.getAbsolutePath(), pig.getPigContext())
+ + Util.generateURI(Util.encodeEscape(in.getAbsolutePath()), pig.getPigContext())
+ "';");
pig.registerQuery("A=foreach (group (filter AA by $0 < '0') all) generate flatten($1);");
- pig.registerQuery("store A into '" + out.getAbsolutePath() + "';");
+ pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutputFilePath) + "';");
FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
pig.getPigContext().getProperties()));
- FSDataInputStream is = fs.open(new Path(out.getAbsolutePath() +
+ FSDataInputStream is = fs.open(new Path(clusterOutputFilePath +
"/part-r-00000.bz2"));
CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
@@ -327,11 +330,11 @@ public class TestBZip {
assertEquals(-1, cis.read(new byte[100]));
cis.close();
- pig.registerQuery("B = load '" + out.getAbsolutePath() + "';");
+ pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutputFilePath) + "';");
pig.openIterator("B");
in.delete();
- out.delete();
+ Util.deleteFile(cluster, clusterOutputFilePath);
}
@@ -587,14 +590,14 @@ public class TestBZip {
compressedInputFileName1);
// pig script to read uncompressed input
- String script = "a = load '" + unCompressedInputFileName +"';";
+ String script = "a = load '" + Util.encodeEscape(unCompressedInputFileName) +"';";
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
.getProperties());
pig.registerQuery(script);
Iterator<Tuple> it1 = pig.openIterator("a");
// pig script to read compressed concatenated input
- script = "a = load '" + compressedInputFileName1 +"';";
+ script = "a = load '" + Util.encodeEscape(compressedInputFileName1) +"';";
pig.registerQuery(script);
Iterator<Tuple> it2 = pig.openIterator("a");
Modified: pig/trunk/test/org/apache/pig/test/TestCombiner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestCombiner.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestCombiner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestCombiner.java Wed Oct 31 21:22:41 2012
@@ -146,10 +146,11 @@ public class TestCombiner {
}
ps.close();
} else {
+ inputFileName = Util.removeColon(inputFileName);
Util.createInputFile(cluster, inputFileName, inputLines.toArray(new String[] {}));
}
pig.registerQuery(loadAlias + " = load '"
- + inputFileName + "' using "
+ + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(',');");
return inputFileName;
}
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java Wed Oct 31 21:22:41 2012
@@ -173,14 +173,15 @@ public class TestEvalPipeline {
t.append(weights);
b.add(t);
- File tmpFile = Util.createTempFileDelOnExit("tmp", "");
- tmpFile.deleteOnExit();
- String fileName = tmpFile.getAbsolutePath();
+ File tmpFile = File.createTempFile("tmp", "");
+ tmpFile.delete(); // we only needed the temp file name, so delete the file
+ String fileName = Util.removeColon(tmpFile.getAbsolutePath());
+
PigFile f = new PigFile(fileName);
f.store(b, new FuncSpec(BinStorage.class.getCanonicalName()),
pigServer.getPigContext());
- pigServer.registerQuery("a = load '" + fileName + "' using BinStorage();");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
Iterator<Tuple> iter = pigServer.openIterator("b");
t = iter.next();
Modified: pig/trunk/test/org/apache/pig/test/TestFinish.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFinish.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFinish.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFinish.java Wed Oct 31 21:22:41 2012
@@ -109,8 +109,9 @@ public class TestFinish {
} else {
pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
f1 = File.createTempFile("test", "txt");
- f1.deleteOnExit();
- inputFileName = f1.getAbsolutePath();
+ f1.delete();
+ inputFileName = Util.removeColon(f1.getAbsolutePath());
+
String input[] = new String[3];
for(int i = 0; i < 3; i++) {
input[i] = ('a'+i + ":1");
@@ -144,7 +145,7 @@ public class TestFinish {
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInMapMR-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','" + expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using " + PigStorage.class.getName() + "(':');");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate MYUDF" + "(*);");
Iterator<Tuple> iter = pigServer.openIterator("b");
while(iter.hasNext()){
@@ -161,7 +162,7 @@ public class TestFinish {
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInReduceMR-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','" + expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using " + PigStorage.class.getName() + "(':');");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("a1 = group a by $1;");
pigServer.registerQuery("b = foreach a1 generate MYUDF" + "(*);");
Iterator<Tuple> iter = pigServer.openIterator("b");
@@ -178,7 +179,7 @@ public class TestFinish {
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInMapLoc-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('LOCAL','" + expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using " + PigStorage.class.getName() + "(':');");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate MYUDF" + "(*);");
pigServer.openIterator("b");
checkAndCleanup(ExecType.LOCAL, expectedFileName, inputFileName);
@@ -190,7 +191,7 @@ public class TestFinish {
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInReduceLoc-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('LOCAL','" + expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using " + PigStorage.class.getName() + "(':');");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("a1 = group a by $1;");
pigServer.registerQuery("b = foreach a1 generate MYUDF" + "(*);");
pigServer.openIterator("b");
Modified: pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestInputOutputFileValidator.java Wed Oct 31 21:22:41 2012
@@ -219,10 +219,8 @@ public class TestInputOutputFileValidato
ctx.getLfs().asElement(fp1.getAbsolutePath());
String path = fp1.getAbsolutePath();
- if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
- path = FileLocalizer.parseCygPath(path, FileLocalizer.STYLE_UNIX);
- ElementDescriptor distribElem = ctx.getDfs().asElement(path) ;
+ ElementDescriptor distribElem = ctx.getDfs().asElement(Util.removeColon(path)) ;
localElem.copy(distribElem, null, false);
@@ -234,10 +232,8 @@ public class TestInputOutputFileValidato
File fp1 = generateTempFile() ;
String path = fp1.getAbsolutePath();
- if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
- path = FileLocalizer.parseCygPath(path, FileLocalizer.STYLE_UNIX);
- ElementDescriptor distribElem = ctx.getDfs().asElement(path) ;
+ ElementDescriptor distribElem = ctx.getDfs().asElement(Util.removeColon(path)) ;
if (distribElem.exists()) {
distribElem.delete() ;
Modified: pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJobSubmission.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJobSubmission.java Wed Oct 31 21:22:41 2012
@@ -91,11 +91,11 @@ public class TestJobSubmission {
public void setUp() throws Exception{
curDir = System.getProperty("user.dir");
inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
- if ((System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")))
- inpDir="/"+FileLocalizer.parseCygPath(inpDir, FileLocalizer.STYLE_WINDOWS);
golDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/GoldenFiles/";
- if ((System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")))
+ if (Util.WINDOWS) {
+ inpDir="/"+FileLocalizer.parseCygPath(inpDir, FileLocalizer.STYLE_WINDOWS);
golDir="/"+FileLocalizer.parseCygPath(golDir, FileLocalizer.STYLE_WINDOWS);
+ }
}
@After
Modified: pig/trunk/test/org/apache/pig/test/TestPigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigContext.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigContext.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigContext.java Wed Oct 31 21:22:41 2012
@@ -184,7 +184,9 @@ public class TestPigContext {
int LOOP_COUNT = 40;
File tmpFile = File.createTempFile("test", "txt");
- tmpFile.deleteOnExit();
+ tmpFile.delete(); // don't actually want the file, just the filename
+ String clusterTmpPath = Util.removeColon(tmpFile.getCanonicalPath());
+
String localInput[] = new String[LOOP_COUNT];
Random r = new Random(1);
int rand;
@@ -192,9 +194,10 @@ public class TestPigContext {
rand = r.nextInt(100);
localInput[i] = Integer.toString(rand);
}
- Util.createInputFile(cluster, tmpFile.getCanonicalPath(), localInput);
+ Util.createInputFile(cluster, clusterTmpPath, localInput);
+
FileLocalizer.deleteTempFiles();
- pigServer.registerQuery("A = LOAD '" + tmpFile.getCanonicalPath() + "' using TestUDF2() AS (num:chararray);");
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterTmpPath)+ "' using TestUDF2() AS (num:chararray);");
pigServer.registerQuery("B = foreach A generate TestUDF1(num);");
Iterator<Tuple> iter = pigServer.openIterator("B");
if(!iter.hasNext()) fail("No output found");
@@ -203,7 +206,7 @@ public class TestPigContext {
assertTrue(t.get(0) instanceof Integer);
assertTrue((Integer)t.get(0) == 1);
}
- Util.deleteFile(cluster, tmpFile.getCanonicalPath());
+ Util.deleteFile(cluster, clusterTmpPath);
Util.deleteDirectory(tmpDir);
}
@@ -265,9 +268,9 @@ public class TestPigContext {
for (final String command : commands) {
pigServer.registerQuery(command);
}
- String outFile = input.getAbsolutePath() + ".out";
- pigServer.store("counts", outFile);
- Util.deleteFile(cluster, outFile);
+ String outFileName = Util.removeColon(input.getAbsolutePath() + ".out");
+ pigServer.store("counts", outFileName);
+ Util.deleteFile(cluster, outFileName);
}
private void check_asserts(PigServer pigServer) {
Modified: pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Wed Oct 31 21:22:41 2012
@@ -410,11 +410,14 @@ public class TestSecondarySort {
ps2.println("1\t4\t4");
ps2.println("2\t3\t1");
ps2.close();
- Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
- Util.copyFromLocalToCluster(cluster, tmpFile2.getCanonicalPath(), tmpFile2.getCanonicalPath());
+ String file1ClusterPath = Util.removeColon(tmpFile1.getCanonicalPath());
+ String file2ClusterPath = Util.removeColon(tmpFile2.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
- pigServer.registerQuery("B = LOAD '" + tmpFile2.getCanonicalPath() + "' AS (b0, b1, b2);");
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), file1ClusterPath);
+ Util.copyFromLocalToCluster(cluster, tmpFile2.getCanonicalPath(), file2ClusterPath);
+
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(file1ClusterPath) + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = LOAD '" + Util.encodeEscape(file2ClusterPath) + "' AS (b0, b1, b2);");
pigServer.registerQuery("C = cogroup A by a0, B by b0 parallel 2;");
pigServer
.registerQuery("D = foreach C { E = limit A 10; F = E.a1; G = DISTINCT F; generate group, COUNT(G);};");
@@ -424,8 +427,8 @@ public class TestSecondarySort {
assertTrue(iter.hasNext());
assertTrue(iter.next().toString().equals("(1,2)"));
assertFalse(iter.hasNext());
- Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
- Util.deleteFile(cluster, tmpFile2.getCanonicalPath());
+ Util.deleteFile(cluster, file1ClusterPath);
+ Util.deleteFile(cluster, file2ClusterPath);
}
@Test
@@ -439,8 +442,11 @@ public class TestSecondarySort {
ps1.println("1\t2\t4");
ps1.println("2\t3\t4");
ps1.close();
- Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+
+ String clusterPath = Util.removeColon(tmpFile1.getCanonicalPath());
+
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterPath);
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterPath) + "' AS (a0, a1, a2);");
pigServer.registerQuery("B = group A by $0 parallel 2;");
pigServer.registerQuery("C = foreach B { D = distinct A; generate group, D;};");
Iterator<Tuple> iter = pigServer.openIterator("C");
@@ -449,7 +455,7 @@ public class TestSecondarySort {
assertTrue(iter.hasNext());
assertTrue(iter.next().toString().equals("(1,{(1,2,3),(1,2,4),(1,3,4)})"));
assertFalse(iter.hasNext());
- Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+ Util.deleteFile(cluster, clusterPath);
}
@Test
@@ -468,16 +474,18 @@ public class TestSecondarySort {
"(2,{(2,3,4)})",
"(1,{(1,2,3),(1,2,4),(1,2,4),(1,2,4),(1,3,4)})"
};
-
- Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+
+ String clusterPath = Util.removeColon(tmpFile1.getCanonicalPath());
+
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterPath);
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterPath) + "' AS (a0, a1, a2);");
pigServer.registerQuery("B = group A by $0 parallel 2;");
pigServer.registerQuery("C = foreach B { D = limit A 10; E = order D by $1; generate group, E;};");
Iterator<Tuple> iter = pigServer.openIterator("C");
Schema s = pigServer.dumpSchema("C");
-
+
Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
- Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+ Util.deleteFile(cluster, clusterPath);
}
@Test
@@ -496,16 +504,18 @@ public class TestSecondarySort {
"(2,{(2,3,4)})",
"(1,{(1,8,4),(1,4,4),(1,3,4),(1,2,3),(1,2,4)})"
};
-
- Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+
+ String clusterPath = Util.removeColon(tmpFile1.getCanonicalPath());
+
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterPath);
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterPath) + "' AS (a0, a1, a2);");
pigServer.registerQuery("B = group A by $0 parallel 2;");
pigServer.registerQuery("C = foreach B { D = order A by a1 desc; generate group, D;};");
Iterator<Tuple> iter = pigServer.openIterator("C");
Schema s = pigServer.dumpSchema("C");
-
+
Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
- Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+ Util.deleteFile(cluster, clusterPath);
}
// @Test
@@ -526,8 +536,8 @@ public class TestSecondarySort {
ps2.close();
Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
Util.copyFromLocalToCluster(cluster, tmpFile2.getCanonicalPath(), tmpFile2.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
- pigServer.registerQuery("B = LOAD '" + tmpFile2.getCanonicalPath() + "' AS (b0, b1, b2);");
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(tmpFile1.getCanonicalPath()) + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = LOAD '" + Util.encodeEscape(tmpFile2.getCanonicalPath()) + "' AS (b0, b1, b2);");
pigServer.registerQuery("C = cogroup A by (a0,a1), B by (b0,b1) parallel 2;");
pigServer.registerQuery("D = ORDER C BY group;");
pigServer.registerQuery("E = foreach D { F = limit A 10; G = ORDER F BY a2; generate group, COUNT(G);};");
@@ -586,8 +596,10 @@ public class TestSecondarySort {
"((1,3),{(3,4)})",
"((2,3),{(3,4)})"
};
- Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+ String clusterFilePath = Util.removeColon(tmpFile1.getCanonicalPath());
+
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterFilePath);
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterFilePath) + "' AS (a0, a1, a2);");
pigServer.registerQuery("B = group A by (a0, a1);");
pigServer.registerQuery("C = foreach B { C1 = A.(a1,a2); generate group, C1;};");
Iterator<Tuple> iter = pigServer.openIterator("C");
Modified: pig/trunk/test/org/apache/pig/test/TestTypedMap.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypedMap.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypedMap.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypedMap.java Wed Oct 31 21:22:41 2012
@@ -44,7 +44,10 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class TestTypedMap {
- private String tmpDirName = System.getProperty("java.io.tmpdir") + "/pigtest/tmp";
+ // this string is used in Pig scripts, where '\' is the escape character.
+ // "\\" has no effect on path resolution in Java, so we escape this here to
+ // fix all the cases that use this to construct paths in this test.
+ private String tmpDirName = Util.encodeEscape(System.getProperty("java.io.tmpdir")) + "/pigtest/tmp";
@Before
public void setUp() throws Exception {
Modified: pig/trunk/test/org/apache/pig/test/TestUDFContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestUDFContext.java?rev=1404358&r1=1404357&r2=1404358&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestUDFContext.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestUDFContext.java Wed Oct 31 21:22:41 2012
@@ -53,9 +53,9 @@ public class TestUDFContext {
File b = Util.createLocalInputFile("b.txt", new String[] { "dumber" });
FileLocalizer.deleteTempFiles();
PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
- String[] statement = { "A = LOAD '" + a.getAbsolutePath() +
+ String[] statement = { "A = LOAD '" + Util.encodeEscape(a.getAbsolutePath()) +
"' USING org.apache.pig.test.utils.UDFContextTestLoader('joe');",
- "B = LOAD '" + b.getAbsolutePath() +
+ "B = LOAD '" + Util.encodeEscape(b.getAbsolutePath()) +
"' USING org.apache.pig.test.utils.UDFContextTestLoader('jane');",
"C = union A, B;",
"D = FOREACH C GENERATE $0, $1, org.apache.pig.test.utils.UDFContextTestEvalFunc($0), " +