You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/12/18 23:55:00 UTC
svn commit: r1552145 - in /pig/trunk: ./ src/org/apache/pig/impl/io/
test/org/apache/pig/builtin/ test/org/apache/pig/test/
Author: daijy
Date: Wed Dec 18 22:54:59 2013
New Revision: 1552145
URL: http://svn.apache.org/r1552145
Log:
PIG-3572: Fix all unit test for during build pig with Hadoop 2.X on Windows
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
pig/trunk/test/org/apache/pig/test/TestAlgebraicInstantiation.java
pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Dec 18 22:54:59 2013
@@ -58,6 +58,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3572: Fix all unit test for during build pig with Hadoop 2.X on Windows (ssvinarchukhorton via daijy)
+
PIG-3633: AvroStorage tests are failing when running against Avro 1.7.5 (jarcec via cheolsoo)
PIG-3612: Storing schema does not work cross cluster with PigStorage and JsonStorage (rohini)
Modified: pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original)
+++ pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Wed Dec 18 22:54:59 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
import org.apache.pig.ExecType;
import org.apache.pig.PigException;
import org.apache.pig.backend.datastorage.ContainerDescriptor;
@@ -486,6 +487,9 @@ public class FileLocalizer {
}
public static String hadoopify(String filename, PigContext pigContext) throws IOException {
+ if (Shell.WINDOWS){
+ filename = filename.replace('\\','/');
+ }
if (filename.startsWith(LOCAL_PREFIX)) {
filename = filename.substring(LOCAL_PREFIX.length());
}
Modified: pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java Wed Dec 18 22:54:59 2013
@@ -69,7 +69,7 @@ public class TestAvroStorage {
final protected static Log LOG = LogFactory.getLog(TestAvroStorage.class);
final private static String basedir = "test/org/apache/pig/builtin/avro/";
- final private static String outbasedir = System.getProperty("user.dir") + "/build/test/TestAvroStorage/";
+ private static String outbasedir = System.getProperty("user.dir") + "/build/test/TestAvroStorage/";
final private static String[] datadir = {
"data/avro",
"data/avro/compressed",
@@ -302,6 +302,9 @@ public class TestAvroStorage {
private String createOutputName() {
final StackTraceElement[] st = Thread.currentThread().getStackTrace();
+ if(Util.WINDOWS){
+ outbasedir = outbasedir.replace('\\','/');
+ }
return outbasedir + st[2].getMethodName();
}
Modified: pig/trunk/test/org/apache/pig/test/TestAlgebraicInstantiation.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestAlgebraicInstantiation.java?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestAlgebraicInstantiation.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestAlgebraicInstantiation.java Wed Dec 18 22:54:59 2013
@@ -32,29 +32,33 @@ import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
public class TestAlgebraicInstantiation {
Boolean[] nullFlags = new Boolean[]{ false, true};
- MiniCluster cluster = MiniCluster.buildCluster();
+ static MiniCluster cluster = MiniCluster.buildCluster();
private PigServer pig;
private File tmpFile;
public TestAlgebraicInstantiation() throws ExecException {
pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
}
-
+
@Before
public void setUp() throws Exception {
-
tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
ps.println("1\t2");
ps.close();
}
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ cluster.shutDown();
+ }
@Test
public void testAlgebraicInstantiation() throws IOException {
Modified: pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java Wed Dec 18 22:54:59 2013
@@ -120,9 +120,12 @@ public class TestJsonLoaderStorage {
FileWriter writer = new FileWriter(tempFile);
writer.write(input);
writer.close();
-
+ String path = tempFile.getAbsolutePath();
+ if (Util.WINDOWS){
+ path = path.replace('\\','/');
+ }
PigServer pigServer = new PigServer(ExecType.LOCAL);
- pigServer.registerQuery("data = load '" + tempFile.getAbsolutePath()
+ pigServer.registerQuery("data = load '" + path
+ "' using JsonLoader('" + schema + "');");
return pigServer.openIterator("data");
@@ -138,11 +141,16 @@ public class TestJsonLoaderStorage {
FileWriter w = new FileWriter(tempInputFile);
w.write(input);
w.close();
-
+ String pathInputFile = tempInputFile.getAbsolutePath();
+ String pathJsonFile = tempJsonFile.getAbsolutePath();
+ if (Util.WINDOWS){
+ pathInputFile = pathInputFile.replace('\\','/');
+ pathJsonFile = pathJsonFile.replace('\\','/');
+ }
PigServer pigServer = new PigServer(ExecType.LOCAL);
- pigServer.registerQuery("data = load '" + tempInputFile.getAbsolutePath()
+ pigServer.registerQuery("data = load '" + pathInputFile
+ "' as (" + schema + ");");
- pigServer.registerQuery("store data into '" + tempJsonFile.getAbsolutePath()
+ pigServer.registerQuery("store data into '" + pathJsonFile
+ "' using JsonStorage();");
tempJsonFile.deleteOnExit();
@@ -274,15 +282,23 @@ public class TestJsonLoaderStorage {
FileWriter w = new FileWriter(tempInputFile);
w.write(rawInput);
w.close();
+ String pattInputFile = tempInputFile.getAbsolutePath();
+ String pattJsonFile = tempJsonFile.getAbsolutePath();
+ String pattJson2File = tempJson2File.getAbsolutePath();
+ if(Util.WINDOWS){
+ pattInputFile = pattInputFile.replace('\\','/');
+ pattJsonFile = pattJsonFile.replace('\\','/');
+ pattJson2File = pattJson2File.replace('\\','/');
+ }
PigServer pigServer = new PigServer(ExecType.LOCAL);
- pigServer.registerQuery("data = load '" + tempInputFile.getAbsolutePath()
+ pigServer.registerQuery("data = load '" + pattInputFile
+ "' as (" + schema + ");");
- pigServer.registerQuery("store data into '" + tempJsonFile.getAbsolutePath()
+ pigServer.registerQuery("store data into '" + pattJsonFile
+ "' using JsonStorage();");
- pigServer.registerQuery("json = load '" + tempJsonFile.getAbsolutePath()
+ pigServer.registerQuery("json = load '" + pattJsonFile
+ "' using JsonLoader('" + schema + "');");
- pigServer.registerQuery("store json into '" + tempJson2File.getAbsolutePath()
+ pigServer.registerQuery("store json into '" + pattJson2File
+ "' using JsonStorage();");
tempJsonFile.deleteOnExit();
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Wed Dec 18 22:54:59 2013
@@ -2185,11 +2185,20 @@ public class TestMacroExpansion {
// Test for PIG-3359
@Test
public void testDeclareInMacroFile() throws Exception {
- String macro =
- "%declare ECHOED_MULTIPLIER `echo \"$MULTIPLIER\"`" +
- "DEFINE MultiplyMacro(numbers) RETURNS multiplied {\n" +
- " $multiplied = FOREACH $numbers GENERATE $0 * $ECHOED_MULTIPLIER;\n" +
- "};";
+ String macro;
+ if(Util.WINDOWS){
+ macro =
+ "%declare ECHOED_MULTIPLIER `echo $MULTIPLIER`" +
+ "DEFINE MultiplyMacro(numbers) RETURNS multiplied {\n" +
+ " $multiplied = FOREACH $numbers GENERATE $0 * $ECHOED_MULTIPLIER;\n" +
+ "};";
+ } else {
+ macro =
+ "%declare ECHOED_MULTIPLIER `echo \"$MULTIPLIER\"`" +
+ "DEFINE MultiplyMacro(numbers) RETURNS multiplied {\n" +
+ " $multiplied = FOREACH $numbers GENERATE $0 * $ECHOED_MULTIPLIER;\n" +
+ "};";
+ }
createFile("my_macro.pig", macro);
String script =
@@ -2267,7 +2276,7 @@ public class TestMacroExpansion {
private void verify(String s, String expected) throws Exception {
createFile("myscript.pig", s);
-
+
String[] args = { "-Dpig.import.search.path=/tmp", "-x", "local", "-c", "myscript.pig" };
PigStats stats = PigRunner.run(args, null);
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1552145&r1=1552144&r2=1552145&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Wed Dec 18 22:54:59 2013
@@ -348,6 +348,9 @@ public class Util {
static public void createInputFile(FileSystem fs, String fileName,
String[] inputData) throws IOException {
+ if(Util.WINDOWS){
+ fileName = fileName.replace('\\','/');
+ }
if(fs.exists(new Path(fileName))) {
throw new IOException("File " + fileName + " already exists on the FileSystem");
}
@@ -362,6 +365,9 @@ public class Util {
}
static public String[] readOutput(FileSystem fs, String fileName) throws IOException {
+ if(Util.WINDOWS){
+ fileName = fileName.replace('\\','/');
+ }
Path path = new Path(fileName);
if(!fs.exists(path)) {
throw new IOException("Path " + fileName + " does not exist on the FileSystem");
@@ -405,6 +411,9 @@ public class Util {
static public OutputStream createInputFile(MiniCluster cluster,
String fileName) throws IOException {
FileSystem fs = cluster.getFileSystem();
+ if(Util.WINDOWS){
+ fileName = fileName.replace('\\','/');
+ }
if (fs.exists(new Path(fileName))) {
throw new IOException("File " + fileName
+ " already exists on the minicluster");
@@ -438,6 +447,9 @@ public class Util {
static public void deleteFile(MiniCluster miniCluster, String fileName)
throws IOException {
FileSystem fs = miniCluster.getFileSystem();
+ if(Util.WINDOWS){
+ fileName = fileName.replace('\\','/');
+ }
fs.delete(new Path(fileName), true);
}
@@ -446,6 +458,9 @@ public class Util {
Configuration conf = ConfigurationUtil.toConfiguration(
pigContext.getProperties());
FileSystem fs = FileSystem.get(conf);
+ if(Util.WINDOWS){
+ fileName = fileName.replace('\\','/');
+ }
fs.delete(new Path(fileName), true);
}
@@ -454,6 +469,9 @@ public class Util {
Configuration conf = ConfigurationUtil.toConfiguration(
pigContext.getProperties());
FileSystem fs = FileSystem.get(conf);
+ if(Util.WINDOWS){
+ fileName = fileName.replace('\\','/');
+ }
return fs.exists(new Path(fileName));
}
@@ -586,6 +604,10 @@ public class Util {
* @throws IOException
*/
static public void copyFromLocalToCluster(MiniCluster cluster, String localFileName, String fileNameOnCluster) throws IOException {
+ if(Util.WINDOWS){
+ localFileName = localFileName.replace('\\','/');
+ fileNameOnCluster = fileNameOnCluster.replace('\\','/');
+ }
PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
String script = getMkDirCommandForHadoop2_0(fileNameOnCluster) + "fs -put " + localFileName + " " + fileNameOnCluster;
@@ -601,6 +623,10 @@ public class Util {
static public void copyFromLocalToLocal(String fromLocalFileName,
String toLocalFileName) throws IOException {
+ if(Util.WINDOWS){
+ fromLocalFileName = fromLocalFileName.replace('\\','/');
+ toLocalFileName = toLocalFileName.replace('\\','/');
+ }
PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
String script = getMkDirCommandForHadoop2_0(toLocalFileName) + "fs -cp " + fromLocalFileName + " " + toLocalFileName;
@@ -618,6 +644,10 @@ public class Util {
}
static public void copyFromClusterToLocal(MiniCluster cluster, String fileNameOnCluster, String localFileName) throws IOException {
+ if(Util.WINDOWS){
+ fileNameOnCluster = fileNameOnCluster.replace('\\','/');
+ localFileName = localFileName.replace('\\','/');
+ }
File parent = new File(localFileName).getParentFile();
if (!parent.exists()) {
parent.mkdirs();
@@ -678,6 +708,9 @@ public class Util {
public static String generateURI(String filename, PigContext context)
throws IOException {
+ if(Util.WINDOWS){
+ filename = filename.replace('\\','/');
+ }
if (context.getExecType() == ExecType.MAPREDUCE) {
return FileLocalizer.hadoopify(filename, context);
} else if (context.getExecType() == ExecType.LOCAL) {