You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/05/13 19:22:42 UTC
svn commit: r1102821 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/scripting/Pig.java
test/org/apache/pig/test/TestScriptLanguage.java
Author: rding
Date: Fri May 13 17:22:42 2011
New Revision: 1102821
URL: http://svn.apache.org/viewvc?rev=1102821&view=rev
Log:
PIG-1827: When passing a parameter to Pig, if the value contains $ it has to be escaped for no apparent reason
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1102821&r1=1102820&r2=1102821&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Fri May 13 17:22:42 2011
@@ -178,6 +178,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1827: When passing a parameter to Pig, if the value contains $ it has to be escaped for no apparent reason (rding)
+
PIG-1938: support project-range as udf argument (thejas)
PIG-2056: Jython error messages should show script name (rding)
Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java?rev=1102821&r1=1102820&r2=1102821&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java Fri May 13 17:22:42 2011
@@ -24,6 +24,7 @@ import java.io.LineNumberReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -276,7 +277,8 @@ public class Pig {
throws IOException {
ArrayList<String> plist = new ArrayList<String>();
for (Entry<String, Object> entry : vars.entrySet()) {
- plist.add(entry.getKey() + "=" + entry.getValue().toString());
+ plist.add(entry.getKey() + "="
+ + fixNonEscapedDollarSign(entry.getValue().toString()));
}
ParameterSubstitutionPreprocessor psp =
@@ -294,6 +296,31 @@ public class Pig {
return writer.toString();
}
+ // Escape the $ so that we can use the parameter substitution
+ // to perform bind operation. Parameter substitution will un-escape $
+ private static String fixNonEscapedDollarSign(String s) {
+ String[] tkns = s.split("\\$", -1);
+
+ if (tkns.length == 1) return s;
+
+ StringBuilder sb = new StringBuilder();
+
+ for (int i = 0; i < tkns.length -1; i++) {
+ if (tkns[i].isEmpty()) {
+ sb.append("\\\\");
+ } else {
+ sb.append(tkns[i]);
+ if (tkns[i].charAt(tkns[i].length()-1) != '\\') {
+ sb.append("\\\\");
+ }
+ }
+ sb.append("$");
+ }
+ sb.append(tkns[tkns.length - 1]);
+
+ return sb.toString();
+ }
+
//-------------------------------------------------------------------------
private static String getScriptFromFile(String filename) throws IOException {
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java?rev=1102821&r1=1102820&r2=1102821&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java Fri May 13 17:22:42 2011
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -29,7 +30,11 @@ import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.scripting.BoundScript;
+import org.apache.pig.scripting.Pig;
import org.apache.pig.scripting.ScriptEngine;
+import org.apache.pig.scripting.ScriptPigContext;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
@@ -357,6 +362,11 @@ public class TestScriptLanguage {
"Pig.fs(\"rmr simple_out\")",
"input = 'simple_table_5'",
"output = 'simple_out'",
+ "testvar = 'abcd$py'",
+ "testvar2 = '$'",
+ "testvar3 = '\\\\\\\\$'",
+ "testvar4 = 'abcd\\$py$'",
+ "testvar5 = 'abcd\\$py'",
"P = Pig.compile(\"\"\"a = load '$input';store a into '$output';\"\"\")",
"Q = P.bind()",
"stats = Q.runSingle()",
@@ -386,7 +396,46 @@ public class TestScriptLanguage {
assertEquals(12, stats.getBytesWritten());
assertEquals(3, stats.getRecordWritten());
}
-
+
+ @Test
+ public void bindLocalVariableTest2() throws Exception {
+ String[] script = {
+ "#!/usr/bin/python",
+ "from org.apache.pig.scripting import *",
+ "Pig.fs(\"rmr simple_out\")",
+ "input = 'bindLocalVariableTest2'",
+ "output = 'simple_out'",
+ "separator = '$'",
+ "P = Pig.compile(\"\"\"a = load '$input' using PigStorage('$separator');store a into '$output';\"\"\")",
+ "Q = P.bind()",
+ "stats = Q.runSingle()",
+ "if stats.isSuccessful():",
+ "\tprint 'success!'",
+ "else:",
+ "\traise 'failed'"
+ };
+ String[] input = {
+ "1$3",
+ "2$4",
+ "3$5"
+ };
+
+ Util.createInputFile(cluster, "bindLocalVariableTest2", input);
+ Util.createLocalInputFile("testScript.py", script);
+
+ ScriptEngine scriptEngine = ScriptEngine.getInstance("jython");
+ Map<String, List<PigStats>> statsMap = scriptEngine.run(pigServer.getPigContext(), "testScript.py");
+ assertEquals(1, statsMap.size());
+ Iterator<List<PigStats>> it = statsMap.values().iterator();
+ PigStats stats = it.next().get(0);
+ assertTrue(stats.isSuccessful());
+ assertEquals(1, stats.getNumberJobs());
+ String name = stats.getOutputNames().get(0);
+ assertEquals("simple_out", name);
+ assertEquals(12, stats.getBytesWritten());
+ assertEquals(3, stats.getRecordWritten());
+ }
+
@Test
public void bindNonStringVariableTest() throws Exception {
String[] script = {
@@ -510,4 +559,26 @@ public class TestScriptLanguage {
String msg = stats.getErrorMessage();
assertEquals(expected, msg.substring(0, expected.length()));
}
+
+ @Test
+ public void testFixNonEscapedDollarSign() throws Exception {
+ java.lang.reflect.Method fixNonEscapedDollarSign = Class.forName(
+ "org.apache.pig.scripting.Pig").getDeclaredMethod(
+ "fixNonEscapedDollarSign", new Class[] { String.class });
+
+ fixNonEscapedDollarSign.setAccessible(true);
+
+ String s = (String)fixNonEscapedDollarSign.invoke(null, "abc$py$");
+ assertEquals("abc\\\\$py\\\\$", s);
+
+ s = (String)fixNonEscapedDollarSign.invoke(null, "$abc$py");
+ assertEquals("\\\\$abc\\\\$py", s);
+
+ s = (String)fixNonEscapedDollarSign.invoke(null, "$");
+ assertEquals("\\\\$", s);
+
+ s = (String)fixNonEscapedDollarSign.invoke(null, "$$abc");
+ assertEquals("\\\\$\\\\$abc", s);
+ }
+
}