You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/09/12 08:13:50 UTC
svn commit: r1522454 - in /pig/trunk: ./ src/org/apache/pig/impl/
src/org/apache/pig/impl/streaming/ src/org/apache/pig/impl/util/
src/org/apache/pig/scripting/groovy/ src/org/apache/pig/tools/parameters/
test/org/apache/pig/newplan/logical/relational/...
Author: daijy
Date: Thu Sep 12 06:13:49 2013
New Revision: 1522454
URL: http://svn.apache.org/r1522454
Log:
PIG-3333: Fix remaining Windows core unit test failures
Added:
pig/trunk/test/org/apache/pig/test/data/generate_date.bat
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java
pig/trunk/src/org/apache/pig/impl/util/JavaCompilerHelper.java
pig/trunk/src/org/apache/pig/impl/util/UriUtil.java
pig/trunk/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java
pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java
pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
pig/trunk/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java
pig/trunk/test/org/apache/pig/test/TestBZip.java
pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java
pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
pig/trunk/test/org/apache/pig/test/TestFinish.java
pig/trunk/test/org/apache/pig/test/TestForEachNestedPlanLocal.java
pig/trunk/test/org/apache/pig/test/TestGrunt.java
pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java
pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
pig/trunk/test/org/apache/pig/test/TestPONegative.java
pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
pig/trunk/test/org/apache/pig/test/TestPigContext.java
pig/trunk/test/org/apache/pig/test/TestPigStorage.java
pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
pig/trunk/test/org/apache/pig/test/TestSample.java
pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
pig/trunk/test/org/apache/pig/test/TestSplit.java
pig/trunk/test/org/apache/pig/test/TestStreaming.java
pig/trunk/test/org/apache/pig/test/TestTmpFileCompression.java
pig/trunk/test/org/apache/pig/test/TestUTF8.java
pig/trunk/test/org/apache/pig/test/TestUnion.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Sep 12 06:13:49 2013
@@ -224,6 +224,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3333: Fix remaining Windows core unit test failures (daijy)
+
PIG-3426: Add support for removing s3 files (jeremykarn via daijy)
PIG-3349: Document ToString(Datetime, String) UDF (cheolsoo)
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Thu Sep 12 06:13:49 2013
@@ -330,7 +330,7 @@ public class PigContext implements Seria
*/
public void addScriptFile(String path) {
if (path != null) {
- aliasedScriptFiles.put(path.replaceFirst("^/", ""), new File(path));
+ aliasedScriptFiles.put(path.replaceFirst("^/", "").replaceAll(":", ""), new File(path));
}
}
@@ -351,7 +351,7 @@ public class PigContext implements Seria
*/
public void addScriptFile(String name, String path) {
if (path != null) {
- aliasedScriptFiles.put(name.replaceFirst("^/", ""), new File(path));
+ aliasedScriptFiles.put(name.replaceFirst("^/", "").replaceAll(":", ""), new File(path));
}
}
Modified: pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java (original)
+++ pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java Thu Sep 12 06:13:49 2013
@@ -34,6 +34,7 @@ import org.apache.pig.builtin.PigStorage
import org.apache.pig.builtin.PigStreaming;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.zookeeper.Shell;
/**
@@ -185,6 +186,10 @@ public class StreamingCommand implements
URI pathUri = null;
URI dfsPath = null;
try {
+ // On Windows, replace "\" into "/"
+ if (Shell.WINDOWS) {
+ path = path.replaceAll("\\\\", "/");
+ }
pathUri = new URI(path);
// Strip away the URI's _fragment_ and _query_
Modified: pig/trunk/src/org/apache/pig/impl/util/JavaCompilerHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/JavaCompilerHelper.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/JavaCompilerHelper.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/JavaCompilerHelper.java Thu Sep 12 06:13:49 2013
@@ -33,6 +33,7 @@ import javax.tools.ToolProvider;
import com.google.common.collect.Lists;
+import org.apache.hadoop.util.Shell;
import org.apache.log4j.Logger;
public class JavaCompilerHelper {
@@ -85,7 +86,7 @@ public class JavaCompilerHelper {
}
public void addToClassPath(String path) {
- this.classPath = this.classPath+":"+path;
+ this.classPath = this.classPath + System.getProperty("path.separator") + path;
}
public String getClassPath() {
Modified: pig/trunk/src/org/apache/pig/impl/util/UriUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/UriUtil.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/UriUtil.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/UriUtil.java Thu Sep 12 06:13:49 2013
@@ -17,6 +17,8 @@
*/
package org.apache.pig.impl.util;
+import java.io.File;
+
public class UriUtil {
public static boolean isHDFSFile(String uri){
@@ -31,7 +33,8 @@ public class UriUtil {
public static boolean isHDFSFileOrLocalOrS3N(String uri){
if(uri == null)
return false;
- if(uri.startsWith("/") || uri.startsWith("hdfs:") || uri.startsWith("file:") || uri.startsWith("s3n:")){
+ if(uri.startsWith("/") || uri.matches("[A-Za-z]:.*") || uri.startsWith("hdfs:")
+ || uri.startsWith("file:") || uri.startsWith("s3n:")){
return true;
}
return false;
Modified: pig/trunk/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java Thu Sep 12 06:13:49 2013
@@ -21,6 +21,7 @@ package org.apache.pig.scripting.groovy;
import groovy.util.ResourceException;
import groovy.util.ScriptException;
+import java.io.File;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Constructor;
@@ -65,7 +66,7 @@ public class GroovyEvalFunc<T> extends E
if (null == c) {
try {
- c = GroovyScriptEngine.getEngine().loadScriptByName(path);
+ c = GroovyScriptEngine.getEngine().loadScriptByName(new File(path).toURI().toString());
} catch (ScriptException se) {
throw new IOException(se);
} catch (ResourceException re) {
Modified: pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java Thu Sep 12 06:13:49 2013
@@ -22,6 +22,7 @@ import groovy.lang.Tuple;
import groovy.util.ResourceException;
import groovy.util.ScriptException;
+import java.io.File;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
@@ -102,7 +103,7 @@ public class GroovyScriptEngine extends
// Load the script
//
- Class c = gse.loadScriptByName(scriptFile);
+ Class c = gse.loadScriptByName(new File(scriptFile).toURI().toString());
//
// Extract the main method
@@ -144,7 +145,7 @@ public class GroovyScriptEngine extends
// Read file
//
- Class c = gse.loadScriptByName(path);
+ Class c = gse.loadScriptByName(new File(path).toURI().toString());
//
// Keep track of initial/intermed/final methods of Albegraic UDFs
Modified: pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Sep 12 06:13:49 2013
@@ -35,6 +35,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
public class PreprocessorContext {
@@ -166,12 +167,17 @@ public class PreprocessorContext {
log.info("Executing command : " + cmd);
// we can't use exec directly since it does not handle
// case like foo -c "bar bar" correctly. It splits on white spaces even in presents of quotes
- String[] cmdArgs = new String[3];
- cmdArgs[0] = "bash";
- cmdArgs[1] = "-c";
- StringBuffer sb = new StringBuffer("exec ");
- sb.append(cmd);
- cmdArgs[2] = sb.toString();
+ StringBuffer sb = new StringBuffer("");
+ String[] cmdArgs;
+ if (Shell.WINDOWS) {
+ cmd = cmd.replaceAll("/", "\\\\");
+ sb.append(cmd);
+ cmdArgs = new String[]{"cmd", "/c", sb.toString() };
+ } else {
+ sb.append("exec ");
+ sb.append(cmd);
+ cmdArgs = new String[]{"bash", "-c", sb.toString() };
+ }
p = Runtime.getRuntime().exec(cmdArgs);
Modified: pig/trunk/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java (original)
+++ pig/trunk/test/org/apache/pig/newplan/logical/relational/TestLocationInPhysicalPlan.java Thu Sep 12 06:13:49 2013
@@ -49,10 +49,10 @@ public class TestLocationInPhysicalPlan
PigServer pigServer = new PigServer(ExecType.LOCAL);
pigServer.setBatchOn();
pigServer.registerQuery(
- "A = LOAD '" + input.getAbsolutePath() + "' using PigStorage();\n"
+ "A = LOAD '" + Util.encodeEscape(input.getAbsolutePath()) + "' using PigStorage();\n"
+ "B = GROUP A BY $0;\n"
+ "A = FOREACH B GENERATE COUNT(A);\n"
- + "STORE A INTO '" + output.getAbsolutePath() + "';");
+ + "STORE A INTO '" + Util.encodeEscape(output.getAbsolutePath()) + "';");
ExecJob job = pigServer.executeBatch().get(0);
List<OriginalLocation> originalLocations = job.getPOStore().getOriginalLocations();
Assert.assertEquals(1, originalLocations.size());
Modified: pig/trunk/test/org/apache/pig/test/TestBZip.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBZip.java Thu Sep 12 06:13:49 2013
@@ -194,6 +194,7 @@ public class TestBZip {
// bzip compressed input
File in = File.createTempFile("junit", ".bz2");
String compressedInputFileName = in.getAbsolutePath();
+ String clusterCompressedFilePath = Util.removeColon(compressedInputFileName);
in.deleteOnExit();
try {
@@ -208,14 +209,14 @@ public class TestBZip {
cos.close();
Util.copyFromLocalToCluster(cluster, compressedInputFileName,
- compressedInputFileName);
+ clusterCompressedFilePath);
// pig script to read compressed input
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
.getProperties());
// pig script to read compressed input
- String script ="a = load '" + compressedInputFileName +"';";
+ String script ="a = load '" + Util.encodeEscape(clusterCompressedFilePath) +"';";
pig.registerQuery(script);
pig.registerQuery("store a into 'intermediate.bz';");
Modified: pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestDefaultDateTimeZone.java Thu Sep 12 06:13:49 2013
@@ -70,7 +70,7 @@ public class TestDefaultDateTimeZone ext
config.setProperty("pig.datetime.default.tz", "+08:00");
PigServer pig = new PigServer(ExecType.LOCAL, config);
pig.registerQuery("a = load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' as (test:datetime);");
pig.registerQuery("b = filter a by test < ToDate('1970-01-04T00:00:00.000');");
Iterator<Tuple> actualItr = pig.openIterator("b");
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java Thu Sep 12 06:13:49 2013
@@ -1131,7 +1131,7 @@ public class TestEvalPipelineLocal {
// See PIG-2970
public void testDescribeDanglingBranch() throws Throwable {
File f1 = createFile(new String[]{"NYSE\tIBM", "NASDAQ\tYHOO", "NASDAQ\tMSFT"});
- pigServer.registerQuery("daily = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ pigServer.registerQuery("daily = load '" + Util.encodeEscape(Util.generateURI(f1.toString(), pigServer.getPigContext()))
+"' as (exchange, symbol);");
pigServer.registerQuery("grpd = group daily by exchange;");
pigServer.registerQuery("unique = foreach grpd { sym = daily.symbol; uniq_sym = distinct sym; uniq_sym = distinct sym; generate group, daily;};");
Modified: pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java Thu Sep 12 06:13:49 2013
@@ -60,7 +60,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using "+PigStorage.class.getName() +"(':');");
String query = "A = filter A by ($0 == $1 and $0 <= $1);";
log.info(query);
@@ -92,7 +92,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 != $1;";
log.info(query);
@@ -120,7 +120,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "' using "
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext())) + "' using "
+ PigStorage.class.getName() + "(':') as (f1: double, f2:double);");
String query = "A = filter A by ($0 > $1 and $0 >= $1);";
@@ -145,7 +145,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext())) + "';");
String query = "A = foreach A generate ($1 >= "+ LOOP_COUNT+"-10?'1':'0');";
log.info(query);
pig.registerQuery(query);
@@ -174,7 +174,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext()) + "';");
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext())) + "';");
String query = "A = foreach A generate (($0 < 10 or $0 < 9)?(($1 >= 5 and $1 >= 4) ? 2: 1) : 0);";
log.info(query);
pig.registerQuery(query);
@@ -204,7 +204,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':') as (a: double, b:double);");
String query = "A = filter A by ($0 <= $1 and $0 < $1);";
@@ -236,7 +236,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by ($0 > $1 or $0 >= $1);";
@@ -267,7 +267,7 @@ public class TestFilterOpNumeric {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':') as (a: double, b:double);");
String query = "A = filter A by ($0 <= $1 or $0 < $1);";
Modified: pig/trunk/test/org/apache/pig/test/TestFilterOpString.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFilterOpString.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFilterOpString.java Thu Sep 12 06:13:49 2013
@@ -68,7 +68,7 @@ public class TestFilterOpString {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 eq $1;";
@@ -107,7 +107,7 @@ public class TestFilterOpString {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 neq $1;";
@@ -146,7 +146,7 @@ public class TestFilterOpString {
}
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 gt $1;";
@@ -190,7 +190,7 @@ public class TestFilterOpString {
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 gte $1;";
@@ -229,7 +229,7 @@ public class TestFilterOpString {
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 lt $1;";
@@ -271,7 +271,7 @@ public class TestFilterOpString {
ps.close();
pig.registerQuery("A=load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 lte $1;";
Modified: pig/trunk/test/org/apache/pig/test/TestFinish.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFinish.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFinish.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFinish.java Thu Sep 12 06:13:49 2013
@@ -141,7 +141,7 @@ public class TestFinish {
String expectedFileName = "testFinishInMapMR-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
+ expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using "
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate MYUDF" + "(*);");
Iterator<Tuple> iter = pigServer.openIterator("b");
@@ -160,7 +160,7 @@ public class TestFinish {
String expectedFileName = "testFinishInReduceMR-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
+ expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using "
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("a1 = group a by $1;");
pigServer.registerQuery("b = foreach a1 generate MYUDF" + "(*);");
@@ -179,7 +179,7 @@ public class TestFinish {
String expectedFileName = "testFinishInMapLoc-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('LOCAL','"
+ expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using "
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate MYUDF" + "(*);");
pigServer.openIterator("b");
@@ -193,7 +193,7 @@ public class TestFinish {
String expectedFileName = "testFinishInReduceLoc-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('LOCAL','"
+ expectedFileName + "');");
- pigServer.registerQuery("a = load '" + inputFileName + "' using "
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("a1 = group a by $1;");
pigServer.registerQuery("b = foreach a1 generate MYUDF" + "(*);");
Modified: pig/trunk/test/org/apache/pig/test/TestForEachNestedPlanLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestForEachNestedPlanLocal.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestForEachNestedPlanLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestForEachNestedPlanLocal.java Thu Sep 12 06:13:49 2013
@@ -103,10 +103,10 @@ public class TestForEachNestedPlanLocal
"({('user3','singapore','user3','usa','10'),('user3','singapore','user3','singapore','20')})",
"({})" });
pig.registerQuery("user = load '"
- + Util.generateURI(tmpFiles[0].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[0].toString(), pig.getPigContext()))
+ "' as (uid, region);");
pig.registerQuery("session = load '"
- + Util.generateURI(tmpFiles[1].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[1].toString(), pig.getPigContext()))
+ "' as (uid, region, duration);");
pig.registerQuery("C = cogroup user by uid, session by uid;");
pig.registerQuery("D = foreach C {"
@@ -131,10 +131,10 @@ public class TestForEachNestedPlanLocal
"({('user3','singapore','user3','singapore','20')})",
"({})" });
pig.registerQuery("user = load '"
- + Util.generateURI(tmpFiles[0].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[0].toString(), pig.getPigContext()))
+ "' as (uid, region);");
pig.registerQuery("session = load '"
- + Util.generateURI(tmpFiles[1].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[1].toString(), pig.getPigContext()))
+ "' as (uid, region, duration);");
pig.registerQuery("C = cogroup user by uid, session by uid;");
pig.registerQuery("D = foreach C {"
@@ -161,13 +161,13 @@ public class TestForEachNestedPlanLocal
"({('user3','singapore','user3','usa','10','user3','user','female'),('user3','singapore','user3','singapore','20','user3','user','female')})",
"({})" });
pig.registerQuery("user = load '"
- + Util.generateURI(tmpFiles[0].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[0].toString(), pig.getPigContext()))
+ "' as (uid, region);");
pig.registerQuery("session = load '"
- + Util.generateURI(tmpFiles[1].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[1].toString(), pig.getPigContext()))
+ "' as (uid, region, duration);");
pig.registerQuery("profile = load '"
- + Util.generateURI(tmpFiles[2].toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFiles[2].toString(), pig.getPigContext()))
+ "' as (uid, role, gender);");
pig.registerQuery("C = cogroup user by uid, session by uid, profile by uid;");
pig.registerQuery("D = foreach C {"
Modified: pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestGrunt.java Thu Sep 12 06:13:49 2013
@@ -77,7 +77,7 @@ public class TestGrunt {
PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
PigContext context = server.getPigContext();
- String strCmd = "copyFromLocal /bin/sh sh_copy ;";
+ String strCmd = "copyFromLocal README.txt sh_copy ;";
ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
InputStreamReader reader = new InputStreamReader(cmd);
@@ -1116,11 +1116,11 @@ public class TestGrunt {
public void testShellCommandOrder() throws Throwable {
PigServer server = new PigServer(ExecType.LOCAL, new Properties());
- String strRemoveRecurse = "rm -rf";
+ String strRemove = "rm";
if (Util.WINDOWS)
{
- strRemoveRecurse = "rd /S";
+ strRemove = "del";
}
File inputFile = File.createTempFile("testInputFile", ".txt");
@@ -1134,7 +1134,7 @@ public class TestGrunt {
PrintWriter pwScript = new PrintWriter(new FileWriter(inputScript));
pwScript.println("a = load '" + Util.encodeEscape(inputFile.getAbsolutePath()) + "';");
pwScript.println("store a into '" + Util.encodeEscape(outputFile.getAbsolutePath()) + "';");
- pwScript.println("sh " + strRemoveRecurse + " " + Util.encodeEscape(inputFile.getAbsolutePath()));
+ pwScript.println("sh " + strRemove + " " + Util.encodeEscape(inputFile.getAbsolutePath()));
pwScript.close();
InputStream inputStream = new FileInputStream(inputScript.getAbsoluteFile());
Modified: pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java Thu Sep 12 06:13:49 2013
@@ -60,7 +60,7 @@ public class TestInfixArithmetic {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "A = foreach (load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName()
+ "(':')) generate $0, $0 + $1, $1;";
log.info(query);
@@ -89,7 +89,7 @@ public class TestInfixArithmetic {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "A = foreach (load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName()
+ "(':')) generate $0, $0 - $1, $1 ;";
log.info(query);
@@ -118,7 +118,7 @@ public class TestInfixArithmetic {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "A = foreach (load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName()
+ "(':')) generate $0, $0 * $1, $1 ;";
log.info(query);
@@ -147,7 +147,7 @@ public class TestInfixArithmetic {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "A = foreach (load '"
- + Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
+ "' using " + PigStorage.class.getName()
+ "(':')) generate $0, $0 / $1, $1 ;";
log.info(query);
Modified: pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestInputOutputMiniClusterFileValidator.java Thu Sep 12 06:13:49 2013
@@ -219,10 +219,8 @@ public class TestInputOutputMiniClusterF
ctx.getLfs().asElement(fp1.getAbsolutePath());
String path = fp1.getAbsolutePath();
- if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
- path = FileLocalizer.parseCygPath(path, FileLocalizer.STYLE_UNIX);
- ElementDescriptor distribElem = ctx.getDfs().asElement(path) ;
+ ElementDescriptor distribElem = ctx.getDfs().asElement(Util.removeColon(path)) ;
localElem.copy(distribElem, null, false);
@@ -234,10 +232,8 @@ public class TestInputOutputMiniClusterF
File fp1 = generateTempFile() ;
String path = fp1.getAbsolutePath();
- if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
- path = FileLocalizer.parseCygPath(path, FileLocalizer.STYLE_UNIX);
- ElementDescriptor distribElem = ctx.getDfs().asElement(path) ;
+ ElementDescriptor distribElem = ctx.getDfs().asElement(Util.removeColon(path)) ;
if (distribElem.exists()) {
distribElem.delete() ;
Modified: pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java Thu Sep 12 06:13:49 2013
@@ -102,10 +102,10 @@ public class TestJsonLoaderStorage {
File interFile = File.createTempFile("tmp", null);
interFile.delete();
- pigServer.registerQuery("a = load '" + inputFile.getCanonicalPath() + "' as (a0:int, a1:chararray, a2, a3:(a30:int));");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFile.getCanonicalPath()) + "' as (a0:int, a1:chararray, a2, a3:(a30:int));");
pigServer.store("a", interFile.getCanonicalPath(), "JsonStorage");
- pigServer.registerQuery("b = load '" + interFile.getCanonicalPath() + "' using JsonLoader();");
+ pigServer.registerQuery("b = load '" + Util.encodeEscape(interFile.getCanonicalPath()) + "' using JsonLoader();");
Iterator<Tuple> iter = pigServer.openIterator("b");
Tuple t = iter.next();
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Thu Sep 12 06:13:49 2013
@@ -334,7 +334,7 @@ public class TestMacroExpansion {
@Test
public void defineTest2() throws Exception {
String macro = "define group_and_count (A) returns B {\n" +
- " DEFINE CMD `stream.pl data.gz` SHIP('"+Util.encodeEscape(command.toString())+"') CACHE('"+Util.encodeEscape(command.toString())+"');\n" +
+ " DEFINE CMD `perl stream.pl data.gz` SHIP('"+Util.encodeEscape(command.toString())+"') CACHE('"+Util.encodeEscape(command.toString())+"');\n" +
" $B = STREAM $A THROUGH CMD;\n" +
"};\n";
Modified: pig/trunk/test/org/apache/pig/test/TestPONegative.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPONegative.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPONegative.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPONegative.java Thu Sep 12 06:13:49 2013
@@ -296,7 +296,7 @@ public class TestPONegative {
PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
File f = Util.createInputFile("tmp", "", new String[] {"a", "b", "c"});
pig.registerQuery("a = load '"
- + Util.generateURI(f.toString(), pig.getPigContext()) + "';");
+ + Util.encodeEscape(Util.generateURI(f.toString(), pig.getPigContext())) + "';");
// -1 is modeled as POnegative with Constant(1)
pig.registerQuery("b = foreach a generate SIZE(-1);");
Iterator<Tuple> it = pig.openIterator("b");
Modified: pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Thu Sep 12 06:13:49 2013
@@ -24,14 +24,18 @@ import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
+import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.PrintWriter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
import org.apache.pig.tools.parameters.ParseException;
import org.junit.Test;
@@ -134,7 +138,8 @@ public class TestParamSubPreproc {
public void testShellCommand() throws Exception{
log.info("Starting test testShellCommand()");
ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(new FileReader(basedir + "/input4.pig"));
+ pigIStream = WithConditionalReplacement(basedir + "/input4.pig", "sh test/org/apache/pig/test/data/generate_date.sh",
+ "test/org/apache/pig/test/data/generate_date.bat", Shell.WINDOWS);
pigOStream = new FileWriter(basedir + "/output1.pig");
String[] arg = null; //{"date=`sh generate_date.sh`"}; //`date \\T`"};
@@ -261,7 +266,8 @@ public class TestParamSubPreproc {
public void testSubstitutionWithinShellCommand() throws Exception{
log.info("Starting test testSubstitutionWithinShellCommand()");
ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(new FileReader(basedir + "/inputSubstitutionWithinShellCommand.pig"));
+ pigIStream = WithConditionalReplacement(basedir + "/inputSubstitutionWithinShellCommand.pig", "sh test/org/apache/pig/test/data/generate_date.sh",
+ "test/org/apache/pig/test/data/generate_date.bat", Shell.WINDOWS);
pigOStream = new FileWriter(basedir + "/output1.pig");
String[] arg = null;
@@ -344,7 +350,8 @@ public class TestParamSubPreproc {
public void testCmdnameAsParamDeclare() throws Exception{
log.info("Starting test testCmdnameAsParamDeclare()");
ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(new FileReader(basedir + "/inputCmdnameAsParamDeclare.pig"));
+ pigIStream = WithConditionalReplacement(basedir + "/inputCmdnameAsParamDeclare.pig", "sh \\$cmd.sh \\$date",
+ "\\$cmd.bat \\$date", Shell.WINDOWS);
pigOStream = new FileWriter(basedir + "/output1.pig");
String[] arg = null;
@@ -984,9 +991,7 @@ public class TestParamSubPreproc {
pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
pigOStream = new FileWriter(basedir + "/output1.pig");
- String[] arg = {"date=`perl -e 'print \"20080228\n20070101\"' | head -n 1`"};
- if (Util.WINDOWS)
- arg[0] = "date=`perl -e 'print \\\"20080228\n20070101\\\"' | head -n 1`";
+ String[] arg = {"date=`perl -e \"print qq@20080228\\n20070101@\" | head -n 1`"};
String[] argFiles = null;
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
@@ -1339,4 +1344,25 @@ public class TestParamSubPreproc {
inExpected.close();
inResult.close();
}
+ @SuppressWarnings("resource")
+ private BufferedReader WithConditionalReplacement(String filename, String orig, String dest, boolean replace) throws IOException {
+ BufferedReader pigOrigIStream = new BufferedReader(new FileReader(filename));
+ BufferedReader result;
+
+ if (replace) {
+ File tmpInputFile = File.createTempFile("tmp", "");
+ PrintWriter tmppw = new PrintWriter(tmpInputFile);
+ String line;
+ while ((line = pigOrigIStream.readLine())!=null) {
+ line = line.replaceAll(orig, dest);
+ tmppw.println(line);
+ }
+ pigOrigIStream.close();
+ tmppw.close();
+ result = new BufferedReader(new FileReader(tmpInputFile));
+ } else {
+ result = pigOrigIStream;
+ }
+ return result;
+ }
}
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/TestPigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigContext.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigContext.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigContext.java Thu Sep 12 06:13:49 2013
@@ -177,7 +177,7 @@ public class TestPigContext {
Util.createInputFile(cluster, clusterTmpPath, localInput);
FileLocalizer.deleteTempFiles();
- pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(tmpFile.getCanonicalPath())
+ pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterTmpPath)
+ "' using TestUDF2() AS (num:chararray);");
pigServer.registerQuery("B = foreach A generate TestUDF1(num);");
Iterator<Tuple> iter = pigServer.openIterator("B");
@@ -198,16 +198,17 @@ public class TestPigContext {
PigContext pc = new PigContext(ExecType.LOCAL, getProperties());
final int n = pc.scriptFiles.size();
pc.addScriptFile("test/path-1824");
- assertEquals("test/path-1824", pc.getScriptFiles().get("test/path-1824").toString());
+ assertEquals("test" + File.separator + "path-1824", pc.getScriptFiles().get("test/path-1824").toString());
assertEquals("script files should not be populated", n, pc.scriptFiles.size());
pc.addScriptFile("path-1824", "test/path-1824");
- assertEquals("test/path-1824", pc.getScriptFiles().get("path-1824").toString());
+ assertEquals("test" + File.separator + "path-1824", pc.getScriptFiles().get("path-1824").toString());
assertEquals("script files should not be populated", n, pc.scriptFiles.size());
// last add wins when using an alias
pc.addScriptFile("path-1824", "test/some/other/path-1824");
- assertEquals("test/some/other/path-1824", pc.getScriptFiles().get("path-1824").toString());
+ assertEquals("test" + File.separator + "some" + File.separator + "other"
+ + File.separator + "path-1824", pc.getScriptFiles().get("path-1824").toString());
assertEquals("script files should not be populated", n, pc.scriptFiles.size());
// clean up
Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Thu Sep 12 06:13:49 2013
@@ -627,7 +627,7 @@ public class TestPigStorage {
File tmpInput = File.createTempFile("tmp", "tmp");
tmpInput.deleteOnExit();
File outFile = new File(parent, "out");
- pig.registerQuery("a = load '"+tmpInput.getAbsolutePath()+"' as (x:int, y:chararray, z:chararray);");
+ pig.registerQuery("a = load '"+Util.encodeEscape(tmpInput.getAbsolutePath())+"' as (x:int, y:chararray, z:chararray);");
pig.store("a", outFile.getAbsolutePath(), "PigStorage('\\t', '-schema')");
File schemaFile = new File(outFile, ".pig_schema");
@@ -639,7 +639,7 @@ public class TestPigStorage {
FileUtils.moveFile(schemaFile, inputSchemaFile);
File inputFile = new File(inputDir, "data");
Util.writeToFile(inputFile, new String[]{"1"});
- pig.registerQuery("a = load '"+inputDir.getAbsolutePath()+"';");
+ pig.registerQuery("a = load '"+Util.encodeEscape(inputDir.getAbsolutePath())+"';");
Iterator<Tuple> it = pig.openIterator("a");
assertTrue(it.hasNext());
assertEquals(tuple(1,null,null), it.next());
Modified: pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java Thu Sep 12 06:13:49 2013
@@ -119,8 +119,7 @@ public class TestRegisteredJarVisibility
@Test()
public void testRegisteredJarVisibility() throws IOException {
- cluster.getFileSystem().copyFromLocalFile(
- new Path("file://" + INPUT_FILE.getAbsolutePath()), new Path(INPUT_FILE.getName()));
+ Util.copyFromLocalToCluster(cluster, INPUT_FILE.getPath(), INPUT_FILE.getName());
PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
String query = "register " + jarFile.getAbsolutePath() + ";\n"
Modified: pig/trunk/test/org/apache/pig/test/TestSample.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSample.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSample.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSample.java Thu Sep 12 06:13:49 2013
@@ -112,7 +112,7 @@ public class TestSample {
@Test
public void testSample_VariableNone() throws Exception {
- verify("a = LOAD '" + tmpfilepath + "'; " +
+ verify("a = LOAD '" + Util.encodeEscape(tmpfilepath) + "'; " +
"b = GROUP a all;" +
"c = FOREACH b GENERATE COUNT(a) AS count;" +
"myid = SAMPLE a (c.count - c.count);", 0, 0);
@@ -120,7 +120,7 @@ public class TestSample {
@Test
public void testSample_VariableAll() throws Exception {
- verify("a = LOAD '" + tmpfilepath + "'; " +
+ verify("a = LOAD '" + Util.encodeEscape(tmpfilepath) + "'; " +
"b = GROUP a all;" +
"c = FOREACH b GENERATE COUNT(a) AS count;" +
"myid = SAMPLE a 1.0 * (c.count / c.count) PARALLEL 2;", DATALEN, DATALEN); // test for PIG-2156
@@ -128,7 +128,7 @@ public class TestSample {
@Test
public void testSample_VariableSome() throws Exception {
- verify("a = LOAD '" + tmpfilepath + "'; " +
+ verify("a = LOAD '" + Util.encodeEscape(tmpfilepath) + "'; " +
"b = GROUP a all;" +
"c = FOREACH b GENERATE COUNT(a) AS count;" +
"myid = SAMPLE a (c.count / (2.0 * c.count) );", DATALEN/3, DATALEN*2/3);
@@ -137,7 +137,7 @@ public class TestSample {
@Test(expected=FrontendException.class)
public void testSampleScalarException() throws IOException {
String query =
- "a = load '" + tmpfilepath + "';" +
+ "a = load '" + Util.encodeEscape(tmpfilepath) + "';" +
"b = sample a $0;" // reference to non scalar context is not allowed
;
Modified: pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java Thu Sep 12 06:13:49 2013
@@ -538,8 +538,7 @@ public class TestScriptLanguage {
assertTrue(stats.getErrorCode() == 1121);
assertTrue(stats.getReturnCode() == PigRunner.ReturnCode.PIG_EXCEPTION);
- String expected = "Python Error. Traceback (most recent call last):\n" +
- " File \"";
+ String expected = "Python Error. Traceback (most recent call last):";
String msg = stats.getErrorMessage();
Util.checkErrorMessageContainsExpected(msg, expected);
Modified: pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestScriptUDF.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestScriptUDF.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestScriptUDF.java Thu Sep 12 06:13:49 2013
@@ -23,6 +23,7 @@ import java.util.Random;
import junit.framework.Assert;
+import org.apache.hadoop.util.Shell;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.BagFactory;
@@ -217,8 +218,9 @@ public class TestScriptUDF{
"def getEnv(envkey):" ,
" return os.getenv(envkey);"
};
+ String userenv = Shell.WINDOWS?"USERNAME":"USER";
String[] input = {
- "USER",
+ userenv,
"PATH"
};
Modified: pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Thu Sep 12 06:13:49 2013
@@ -380,10 +380,12 @@ public class TestSecondarySort {
ps2.println("1\t4\t4");
ps2.println("2\t3\t1");
ps2.close();
- Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
- Util.copyFromLocalToCluster(cluster, tmpFile2.getCanonicalPath(), tmpFile2.getCanonicalPath());
- pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(tmpFile1.getCanonicalPath()) + "' AS (a0, a1, a2);");
- pigServer.registerQuery("B = LOAD '" + Util.encodeEscape(tmpFile2.getCanonicalPath()) + "' AS (b0, b1, b2);");
+ String clusterPath1 = tmpFile1.getName();
+ String clusterPath2 = tmpFile2.getName();
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterPath1);
+ Util.copyFromLocalToCluster(cluster, tmpFile2.getCanonicalPath(), clusterPath2);
+ pigServer.registerQuery("A = LOAD '" + clusterPath1 + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = LOAD '" + clusterPath2 + "' AS (b0, b1, b2);");
pigServer.registerQuery("C = cogroup A by (a0,a1), B by (b0,b1) parallel 2;");
pigServer.registerQuery("D = ORDER C BY group;");
pigServer.registerQuery("E = foreach D { F = limit A 10; G = ORDER F BY a2; generate group, COUNT(G);};");
@@ -397,8 +399,8 @@ public class TestSecondarySort {
assertTrue(iter.hasNext());
assertEquals("((2,3),1)", iter.next().toString());
assertFalse(iter.hasNext());
- Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
- Util.deleteFile(cluster, tmpFile2.getCanonicalPath());
+ Util.deleteFile(cluster, clusterPath1);
+ Util.deleteFile(cluster, clusterPath2);
}
@Test
@@ -453,7 +455,7 @@ public class TestSecondarySort {
Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
- Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+ Util.deleteFile(cluster, clusterFilePath);
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestSplit.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSplit.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSplit.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSplit.java Thu Sep 12 06:13:49 2013
@@ -49,7 +49,7 @@ public class TestSplit {
@Test
public void testSplit1() throws IOException {
String query =
- "a = load '" + file.getAbsolutePath() + "' as (id:int);" +
+ "a = load '" + Util.encodeEscape(file.getAbsolutePath()) + "' as (id:int);" +
"split a into b if id > 3, c if id < 3, d otherwise;"
;
@@ -63,7 +63,7 @@ public class TestSplit {
@Test
public void testSplit2() throws IOException {
String query =
- "a = load '" + file.getAbsolutePath() + "' as (id:int);" +
+ "a = load '" + Util.encodeEscape(file.getAbsolutePath()) + "' as (id:int);" +
"split a into b if id % 2 == 0, d otherwise;"
;
@@ -80,7 +80,7 @@ public class TestSplit {
"define split_into_two (A,key) returns B, C {" +
" SPLIT $A INTO $B IF $key<4, $C OTHERWISE;" +
"};" +
- "a = load '" + file.getAbsolutePath() + "' as (id:int);" +
+ "a = load '" + Util.encodeEscape(file.getAbsolutePath()) + "' as (id:int);" +
"B, C = split_into_two(a, id);"
;
@@ -98,7 +98,7 @@ public class TestSplit {
@Test(expected=FrontendException.class)
public void testSplitNondeterministic() throws IOException {
String query =
- "a = load '" + file.getAbsolutePath() + "' as (id:int);" +
+ "a = load '" + Util.encodeEscape(file.getAbsolutePath()) + "' as (id:int);" +
"split a into b if RANDOM() < 0.5, d otherwise;"
;
Modified: pig/trunk/test/org/apache/pig/test/TestStreaming.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestStreaming.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestStreaming.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestStreaming.java Thu Sep 12 06:13:49 2013
@@ -852,7 +852,7 @@ public class TestStreaming {
// Pig query to run
pigServer.registerQuery(
- "define CMD `" + command.getName() + " foo` " +
+ "define CMD `perl " + command.getName() + " foo` " +
"ship ('" + Util.encodeEscape(command.toString()) + "') " +
"input('foo' using " + PigStreaming.class.getName() + "()) " +
"output(stdout using " + PigStreaming.class.getName() + "(',')) " +
Modified: pig/trunk/test/org/apache/pig/test/TestTmpFileCompression.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTmpFileCompression.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTmpFileCompression.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTmpFileCompression.java Thu Sep 12 06:13:49 2013
@@ -392,6 +392,7 @@ public class TestTmpFileCompression {
}
assertEquals("Last value does not match",
curval, LOOP_SIZE );
+ reader.close();
} finally {
tFile.delete();
}
Modified: pig/trunk/test/org/apache/pig/test/TestUTF8.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestUTF8.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestUTF8.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestUTF8.java Thu Sep 12 06:13:49 2013
@@ -58,7 +58,7 @@ public class TestUTF8 {
pw.close();
pigServer.registerQuery("a = load '"
- + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ + Util.encodeEscape(Util.generateURI(f1.toString(), pigServer.getPigContext()))
+ "' using " + PigStorage.class.getName() + "();");
Iterator<Tuple> iter = pigServer.openIterator("a");
@@ -91,7 +91,7 @@ public class TestUTF8 {
pw.close();
pigServer.registerQuery("a = load '"
- + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ + Util.encodeEscape(Util.generateURI(f1.toString(), pigServer.getPigContext()))
+ "' using " + PigStorage.class.getName() + "();");
pigServer.registerQuery("b = filter a by $0 == 'ä¸æ';");
Iterator<Tuple> iter = pigServer.openIterator("a");
Modified: pig/trunk/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestUnion.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestUnion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestUnion.java Thu Sep 12 06:13:49 2013
@@ -198,10 +198,10 @@ public class TestUnion {
File f2 = Util.createInputFile("tmp", "b.txt", new String[] {"7\t8\t9", "1\t200\t300"});
File f3 = Util.createInputFile("tmp", "c.txt", new String[] {"1\t20\t30"});
//FileLocalizer.deleteTempFiles();
- pigServer.registerQuery("a = load '" + f1.getAbsolutePath() + "' ;");
- pigServer.registerQuery("b = load '" + f2.getAbsolutePath() + "';");
+ pigServer.registerQuery("a = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' ;");
+ pigServer.registerQuery("b = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "';");
pigServer.registerQuery("c = union a, b;");
- pigServer.registerQuery("d = load '" + f3.getAbsolutePath() + "' ;");
+ pigServer.registerQuery("d = load '" + Util.encodeEscape(f3.getAbsolutePath()) + "' ;");
pigServer.registerQuery("e = cogroup c by $0 inner, d by $0 inner;");
pigServer.explain("e", System.err);
// output should be
@@ -225,8 +225,8 @@ public class TestUnion {
public void testSchemaMergeWithBag() throws Exception {
File f1 = Util.createInputFile("tmp", "input1.txt", new String[] {"dummy"});
File f2 = Util.createInputFile("tmp", "input2.txt", new String[] {"dummy"});
- Util.registerMultiLineQuery(pigServer, "a = load '" + f1.getAbsolutePath() + "';" +
- "b = load '" + f2.getAbsolutePath() + "';" +
+ Util.registerMultiLineQuery(pigServer, "a = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "';" +
+ "b = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "';" +
"c = foreach a generate 1, {(1, 'str1')};" +
"d = foreach b generate 2, {(2, 'str2')};" +
"e = union c,d;" +
@@ -260,8 +260,8 @@ public class TestUnion {
File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"bbb\t222"});
PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
- ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a,b);");
- ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' as (a,b);");
+ ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a,b);");
+ ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' as (a,b);");
ps.registerQuery("C = union A,B;");
ps.registerQuery("D = foreach C generate (chararray)a as a,(int)b as b;");
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1522454&r1=1522453&r2=1522454&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Thu Sep 12 06:13:49 2013
@@ -323,7 +323,8 @@ public class Util {
PrintWriter pw = new PrintWriter(new OutputStreamWriter(new
FileOutputStream(f), "UTF-8"));
for (int i=0; i<inputData.length; i++){
- pw.println(inputData[i]);
+ pw.print(inputData[i]);
+ pw.print("\n");
}
pw.close();
}
@@ -353,7 +354,8 @@ public class Util {
FSDataOutputStream stream = fs.create(new Path(fileName));
PrintWriter pw = new PrintWriter(new OutputStreamWriter(stream, "UTF-8"));
for (int i=0; i<inputData.length; i++){
- pw.println(inputData[i]);
+ pw.print(inputData[i]);
+ pw.print("\n");
}
pw.close();
@@ -861,8 +863,8 @@ public class Util {
}
Process cmdProc = Runtime.getRuntime().exec(cmd);
ProcessReturnInfo pri = new ProcessReturnInfo();
- pri.stdoutContents = getContents(cmdProc.getInputStream());
pri.stderrContents = getContents(cmdProc.getErrorStream());
+ pri.stdoutContents = getContents(cmdProc.getInputStream());
cmdProc.waitFor();
pri.exitCode = cmdProc.exitValue();
return pri;
Added: pig/trunk/test/org/apache/pig/test/data/generate_date.bat
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/generate_date.bat?rev=1522454&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/generate_date.bat (added)
+++ pig/trunk/test/org/apache/pig/test/data/generate_date.bat Thu Sep 12 06:13:49 2013
@@ -0,0 +1,6 @@
+@echo off
+if "%1"=="" (
+ echo 20080228
+) else (
+ echo %1
+)