You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by jc...@apache.org on 2012/10/19 02:05:42 UTC
svn commit: r1399932 - in /pig/branches/branch-0.11: CHANGES.txt
src/org/apache/pig/tools/parameters/PreprocessorContext.java
test/org/apache/pig/test/TestParamSubPreproc.java
Author: jcoveney
Date: Fri Oct 19 00:05:41 2012
New Revision: 1399932
URL: http://svn.apache.org/viewvc?rev=1399932&view=rev
Log:
PIG-2931: $ signs in the replacement string make parameter substitution fail (cheolsoo via jcoveney)
Modified:
pig/branches/branch-0.11/CHANGES.txt
pig/branches/branch-0.11/src/org/apache/pig/tools/parameters/PreprocessorContext.java
pig/branches/branch-0.11/test/org/apache/pig/test/TestParamSubPreproc.java
Modified: pig/branches/branch-0.11/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/CHANGES.txt?rev=1399932&r1=1399931&r2=1399932&view=diff
==============================================================================
--- pig/branches/branch-0.11/CHANGES.txt (original)
+++ pig/branches/branch-0.11/CHANGES.txt Fri Oct 19 00:05:41 2012
@@ -22,6 +22,8 @@ Release 0.11.0 (unreleased)
INCOMPATIBLE CHANGES
+PIG-2931: $ signs in the replacement string make parameter substitution fail (cheolsoo via jcoveney)
+
PIG-1891 Enable StoreFunc to make intelligent decision based on job success or failure (initialcontext via gates)
IMPROVEMENTS
Modified: pig/branches/branch-0.11/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1399932&r1=1399931&r2=1399932&view=diff
==============================================================================
--- pig/branches/branch-0.11/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/branches/branch-0.11/src/org/apache/pig/tools/parameters/PreprocessorContext.java Fri Oct 19 00:05:41 2012
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-/**
+/**
* This is helper class for parameter substitution
*/
@@ -35,18 +35,18 @@ import java.util.regex.Pattern;
public class PreprocessorContext {
private Hashtable<String , String> param_val ;
-
+
private final Log log = LogFactory.getLog(getClass());
/**
* @param limit - max number of parameters. Passing
* smaller number only impacts performance
- */
+ */
public PreprocessorContext(int limit){
param_val = new Hashtable<String, String> (limit);
}
-
- /*
+
+ /*
public void processLiteral(String key, String val) {
processLiteral(key, val, true);
} */
@@ -107,7 +107,7 @@ public class PreprocessorContext {
String sub_val = substitute(val);
sub_val = executeShellCommand(sub_val);
param_val.put(key, sub_val);
- }
+ }
/**
* This method generates value for the specified key by
@@ -129,13 +129,13 @@ public class PreprocessorContext {
String sub_val = substitute(val);
param_val.put(key, sub_val);
- }
+ }
/*
* executes the 'cmd' in shell and returns result
*/
- private String executeShellCommand (String cmd)
+ private String executeShellCommand (String cmd)
{
Process p;
String streamData="";
@@ -188,7 +188,7 @@ public class PreprocessorContext {
} finally {
if (br != null) try {br.close();} catch(Exception e) {}
}
-
+
try {
InputStreamReader isr = new InputStreamReader(p.getErrorStream());
br = new BufferedReader(isr);
@@ -211,14 +211,14 @@ public class PreprocessorContext {
}
private Pattern id_pattern = Pattern.compile("\\$[_]*[a-zA-Z][a-zA-Z_0-9]*");
-
+
public String substitute(String line) {
int index = line.indexOf('$');
if (index == -1) return line;
String replaced_line = line;
-
+
Matcher keyMatcher = id_pattern.matcher( line );
String key="";
String val="";
@@ -232,8 +232,10 @@ public class PreprocessorContext {
throw new RuntimeException("Undefined parameter : "+key);
}
val = param_val.get(key);
- //String litVal = Matcher.quoteReplacement(val);
- replaced_line = replaced_line.replaceFirst("\\$"+key, val);
+ if (val.contains("$")) {
+ val = val.replaceAll("\\$", "\\\\\\$");
+ }
+ replaced_line = replaced_line.replaceFirst("\\$"+key, val);
}
}
Modified: pig/branches/branch-0.11/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1399932&r1=1399931&r2=1399932&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/branches/branch-0.11/test/org/apache/pig/test/TestParamSubPreproc.java Fri Oct 19 00:05:41 2012
@@ -1482,6 +1482,52 @@ public class TestParamSubPreproc extends
log.info("Done");
}
+ /* Test case
+ * Test that $ signs in substitution value are treated as literal replacement strings.
+ */
+ @Test
+ public void testSubstitutionWithDollarSign() throws Exception{
+ try {
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir + "/inputDollarSign.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"filter=\"($0 == 'x') and ($1 == 'y')\""};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDollarSign.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
+ } catch (ParseException e) {
+ fail ("Got ParseException : " + e.getMessage());
+ } catch (RuntimeException e) {
+ fail ("Got RuntimeException : " + e.getMessage());
+ } catch (Error e) {
+ fail ("Got error : " + e.getMessage());
+ }
+ }
+
@Test
public void testMacroDef() throws Exception{
log.info("Starting test testMacroDef() ...");