You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2016/05/13 01:45:16 UTC
svn commit: r1743590 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/parameters/PigFileParser.jj
test/org/apache/pig/test/TestParamSubPreproc.java
Author: knoguchi
Date: Fri May 13 01:45:16 2016
New Revision: 1743590
URL: http://svn.apache.org/viewvc?rev=1743590&view=rev
Log:
PIG-4887: Parameter substitution skipped with glob on register (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1743590&r1=1743589&r2=1743590&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 13 01:45:16 2016
@@ -123,6 +123,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4887: Parameter substitution skipped with glob on register (knoguchi)
+
PIG-4889: Replacing backslash fails as lexical error (knoguchi)
PIG-4880: Overlapping of parameter substitution names inside&outside a macro fails with NPE (knoguchi)
Modified: pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=1743590&r1=1743589&r2=1743590&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Fri May 13 01:45:16 2016
@@ -259,8 +259,11 @@ TOKEN :
<PIGDEFAULT: "%default" >
}
+
TOKEN :
{
+ <REGISTER: "register"> : IN_REGISTER
+ |
<IDENTIFIER: (<SPECIALCHAR>)*<LETTER>(<DIGIT> | <LETTER> | <SPECIALCHAR>)*>
|
<LITERAL: ("\"" ((~["\""])*("\\\"")?)* "\"")|("'" ((~["'"])*("\\\'")?)* "'") >
@@ -276,7 +279,14 @@ TOKEN :
<OTHER: (~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r", "%", "/", "-", "$"])+ >
|
<NOT_OTHER_CHAR: ["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r", "%", "/", "-", "$"] >
-
+}
+
+<IN_REGISTER> MORE : { " " | "\t" | "\r" | "\n"}
+
+<IN_REGISTER> TOKEN: {
+ <PATH: (~["(", ")", ";", "\r", " ", "\t", "\n"])+> {
+ matchedToken.image = image.toString();
+ }: DEFAULT
}
void Parse() throws IOException : {}
@@ -288,6 +298,7 @@ void input() throws IOException :
{
String s;
Token strTok = null;
+ Token strTok2 = null;
}
{
strTok = <PIG>
@@ -308,6 +319,20 @@ void input() throws IOException :
{ pc.validate(strTok.toString()); }
)
|
+ strTok = <REGISTER>
+ strTok2 = <PATH> {}
+ {
+ // Adding a special case for register since it handles "/*" globbing
+ // and this conflicts with general multi-line comment "/* */".
+ // See the comment above on OTHERS on how tokenizer matches the longest
+ // match. Here, string next to "register" is treated as PATH TOKEN
+ // and therefore not consider "/*" as part of the comment
+ // (and avoid the longest match problem).
+ out.append(strTok.image);
+ String sub_line = pc.substitute(strTok2.image);
+ out.append(sub_line);
+ }
+ |
s = paramString(){}
{
//process an ordinary pig line - perform substitution
Modified: pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1743590&r1=1743589&r2=1743590&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Fri May 13 01:45:16 2016
@@ -52,24 +52,9 @@ public class TestParamSubPreproc {
private FileInputStream pigExResultStream;
private String basedir = "test/org/apache/pig/test/data";
- /* Test case 1
- * Use a parameter within a pig script and provide value on the command line.
- */
- @Test
- public void testCmdlineParam() throws Exception{
- log.info("Starting test testCmdlineParam() ...");
- ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
- pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
- pigOStream = new FileWriter(basedir + "/output1.pig");
-
- String[] arg = {"date=20080228"};
- String[] argFiles = null;
- ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
-
- FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+ private void compareResults(InputStream expected, InputStream result) throws IOException {
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(result));
String exLine;
String resLine;
@@ -89,6 +74,26 @@ public class TestParamSubPreproc {
inExpected.close();
inResult.close();
+ }
+
+ /* Test case 1
+ * Use a parameter within a pig script and provide value on the command line.
+ */
+ @Test
+ public void testCmdlineParam() throws Exception{
+ log.info("Starting test testCmdlineParam() ...");
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"date=20080228"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
@@ -110,27 +115,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -152,27 +138,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDefault.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -236,27 +203,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -280,27 +228,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
log.info("Done");
}
@@ -321,27 +251,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultCmdLnPriorDeclare.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -364,27 +275,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
log.info("Done");
}
@@ -406,27 +299,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
log.info("Done");
}
@@ -446,27 +321,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
log.info("Done");
}
@@ -487,27 +344,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
log.info("Done");
}
@@ -527,27 +366,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
}
/* Test case 15,16
@@ -615,27 +436,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 21
@@ -654,27 +456,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
@@ -693,27 +476,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
+ compareResults(pigExResultStream, pigResultStream);
- inExpected.close();
- inResult.close();
}
/* Test case 23
@@ -732,27 +497,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 24
@@ -771,27 +517,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultMulDecs.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 25
@@ -809,27 +536,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 26
@@ -847,27 +555,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
@@ -886,27 +575,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 29
@@ -924,27 +594,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 30
@@ -962,27 +613,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult2.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case 31
@@ -1001,27 +633,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -1042,27 +655,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/inputNoVars.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
+ compareResults(pigExResultStream, pigResultStream);
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
log.info("Done");
}
@@ -1083,27 +678,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult3.txt");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -1123,27 +699,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultComment.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
/* Test case
@@ -1212,27 +769,7 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output26.pig");
InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(expected));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(expected, pigResultStream);
log.info("Done");
}
@@ -1252,27 +789,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDollarSign.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
}
@Test
@@ -1289,28 +807,8 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult6.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
-
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
log.info("Done");
}
@@ -1326,27 +824,9 @@ public class TestParamSubPreproc {
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
pigExResultStream = new FileInputStream(basedir + "/ExpectedResult7.pig");
- BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
- BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
- String exLine;
- String resLine;
- int lineNum=0;
-
- while (true) {
- lineNum++;
- exLine = inExpected.readLine();
- resLine = inResult.readLine();
- if (exLine==null || resLine==null)
- break;
- assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
- }
- if (!(exLine==null && resLine==null)) {
- fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
- }
-
- inExpected.close();
- inResult.close();
+ compareResults(pigExResultStream, pigResultStream);
+ log.info("Done");
}
@Test
@@ -1398,6 +878,71 @@ public class TestParamSubPreproc {
assertEquals(resultContent, "daniel\t10\njenny\t20\n");
}
+ /*
+ * Test parameter substition when register contains /* globbing
+ */
+ @Test
+ public void testSubstitutionWithRegisterGlobbing() throws Exception{
+ log.info("Starting test testSubstitutionWithRegisterGlobbing()");
+ final String queryString =
+ "register /abc/$regdir/*.jar;\n" +
+ "A = LOAD '$input' USING PigStorage ();\n" +
+ "STORE A INTO '$output';\n" +
+ " /* comment that would make register globbing to be part of the multi-line comment */\n";
+
+
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(
+ new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"input = 'input.txt'", "output = 'output.txt'", "regdir = 'def'"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+
+ String expectedString = queryString.replaceAll("\\$input","input.txt")
+ .replaceAll("\\$output","output.txt")
+ .replaceAll("\\$regdir","def");
+ InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
+
+ compareResults(expected, pigResultStream);
+
+ log.info("Done");
+ }
+
+ /*
+ * Test parameter substition when load contains /* globbing
+ */
+ @Test
+ public void testSubstitutionWithLoadGlobbing() throws Exception{
+ log.info("Starting test testSubstitutionWithLoadGlobbing()");
+ final String queryString =
+ "A = LOAD '/zzz/*' USING PigStorage ();\n" +
+ "STORE A INTO '$output';\n" +
+ " /* comment that would make register globbing to be part of the multi-line comment */\n";
+
+
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(
+ new InputStreamReader(new ByteArrayInputStream(queryString.getBytes("UTF-8"))));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"output = 'output.txt'"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+
+ String expectedString = queryString.replaceAll("\\$output","output.txt");
+ InputStream expected = new ByteArrayInputStream(expectedString.getBytes("UTF-8"));
+
+ compareResults(expected, pigResultStream);
+
+ log.info("Done");
+ }
+
@SuppressWarnings("resource")
private BufferedReader WithConditionalReplacement(String filename, String orig, String dest, boolean replace) throws IOException {
BufferedReader pigOrigIStream = new BufferedReader(new FileReader(filename));