You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2016/05/12 21:47:37 UTC
svn commit: r1743583 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
test/org/apache/pig/test/TestPigScriptParser.java
Author: knoguchi
Date: Thu May 12 21:47:37 2016
New Revision: 1743583
URL: http://svn.apache.org/viewvc?rev=1743583&view=rev
Log:
PIG-4889: Replacing backslash fails as lexical error (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1743583&r1=1743582&r2=1743583&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 12 21:47:37 2016
@@ -123,6 +123,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4889: Replacing backslash fails as lexical error (knoguchi)
+
PIG-4880: Overlapping of parameter substitution names inside&outside a macro fails with NPE (knoguchi)
PIG-4881: TestBuiltin.testUniqueID failing on hadoop-1.x (knoguchi)
Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1743583&r1=1743582&r2=1743583&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu May 12 21:47:37 2016
@@ -23,6 +23,7 @@ options {
STATIC = false;
// Case is ignored in keywords
IGNORE_CASE = true;
+ // DEBUG_PARSER = true;
JAVA_UNICODE_ESCAPE = true;
}
@@ -292,7 +293,8 @@ TOKEN_MGR_DECLS : {
<IN_STRING> MORE :
{
- <"\\'">
+ <"\\\\">
+| <"\\'">
| <"'"> { SwitchTo(prevState);}
| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
| <(~[])>
Modified: pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java?rev=1743583&r1=1743582&r2=1743583&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java Thu May 12 21:47:37 2016
@@ -30,6 +30,9 @@ import java.util.Properties;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
+import org.apache.pig.builtin.mock.Storage;
+import org.apache.pig.builtin.mock.Storage.Data;
+import static org.apache.pig.builtin.mock.Storage.tuple;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -156,6 +159,79 @@ public class TestPigScriptParser {
}
}
+ @Test
+ public void testBackSlashOnly() throws Exception {
+ PigServer pig = new PigServer(Util.getLocalTestMode());
+ Data data = Storage.resetData(pig);
+ data.set("input", tuple("abc"), tuple("\\bcd"), tuple("'cde"), tuple("def\\\\"));
+
+ String query =
+ "A = load 'input' USING mock.Storage() as (a0:chararray);\n"
+ // java String is escaping "\" so the following line is equivalent of
+ // B = FILTER A by STARTSWITH(a0,'\\'); in the pig script
+ + "B = FILTER A by STARTSWITH(a0,'\\\\');\n"
+ + "store B into 'out' using mock.Storage;" ;
+
+ Util.registerMultiLineQuery(pig, query);
+ List<Tuple> list = data.get("out");
+
+ assertEquals("There should be only one match", 1, list.size());
+ Tuple t = list.get(0);
+ assertEquals("result should have only one field", 1, t.size() );
+ assertEquals("\\bcd",(String) t.get(0));
+ }
+
+
+ @Test
+ public void testBackSlashSingleQuote() throws Exception {
+ PigServer pig = new PigServer(Util.getLocalTestMode());
+ Data data = Storage.resetData(pig);
+ data.set("input", tuple("abc"), tuple("\\bcd"), tuple("'cde"), tuple("def\\\\"));
+
+ String query =
+ "A = load 'input' USING mock.Storage() as (a0:chararray);\n"
+ // java String is escaping "\" so the following line is equivalent of
+ // B = FILTER A by STARTSWITH(a0,'\''); in the pig script
+ + "B = FILTER A by STARTSWITH(a0,'\\'');\n"
+ + "store B into 'out' using mock.Storage;" ;
+
+ Util.registerMultiLineQuery(pig, query);
+ List<Tuple> list = data.get("out");
+
+ assertEquals("There should be only one match", 1, list.size());
+ Tuple t = list.get(0);
+ assertEquals("result should have only one field", 1, t.size() );
+ assertEquals("'cde",(String) t.get(0));
+ }
+
+ @Test
+ public void testBackSlashReplace() throws Exception {
+ PigServer pig = new PigServer(Util.getLocalTestMode());
+ Data data = Storage.resetData(pig);
+ //After java escaping, these tuples have
+ //'abc', '\bcd' and 'def\\' respectively
+ data.set("input", tuple("abc"), tuple("\\bcd"), tuple("def\\\\"));
+
+ String query =
+ "A = load 'input' USING mock.Storage() as (a0:chararray);\n"
+ // java String is escaping "\" so the following line is equivalent of
+ //"B = FOREACH A GENERATE REPLACE(a0,'\\\\','+');\n"
+ + "B = FOREACH A GENERATE REPLACE(a0,'\\\\\\\\','+');\n"
+ + "store B into 'out' using mock.Storage;" ;
+
+ // REPLACE(a0,'\\\\','+')
+ // --> Pig parser unescape and pass "\\" to REPLACE UDF.
+ // --> REPLACE UDF calls, Pattern.compile("\\"); which
+ // matches "\"
+
+ Util.registerMultiLineQuery(pig, query);
+ List<Tuple> list = data.get("out");
+
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {"('abc')","('+bcd')", "('def++')"});
+ Util.checkQueryOutputsAfterSort(list, expectedRes);
+ }
private void checkParsedConstContent(PigServer pigServer,
PigContext pigContext,
String query,