You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by jc...@apache.org on 2013/01/17 01:27:22 UTC
svn commit: r1434521 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/tools/parameters/PreprocessorContext.java
test/org/apache/pig/test/TestParamSubPreproc.java
test/org/apache/pig/test/data/ExpectedResult7.pig
test/org/apache/pig/test/data/input7.pig
Author: jcoveney
Date: Thu Jan 17 00:27:22 2013
New Revision: 1434521
URL: http://svn.apache.org/viewvc?rev=1434521&view=rev
Log:
PIG-2788: improved string interpolation of variables (jcoveney)
Added:
pig/trunk/test/org/apache/pig/test/data/ExpectedResult7.pig
pig/trunk/test/org/apache/pig/test/data/input7.pig
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1434521&r1=1434520&r2=1434521&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jan 17 00:27:22 2013
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2788: improved string interpolation of variables (jcoveney)
+
PIG-2362: Rework Ant build.xml to use macrodef instead of antcall (azaroth via cheolsoo)
PIG-2857: Add a -tagPath option to PigStorage (prkommireddi via cheolsoo)
Modified: pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1434521&r1=1434520&r2=1434521&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Jan 17 00:27:22 2013
@@ -22,16 +22,16 @@
package org.apache.pig.tools.parameters;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Logger;
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.BufferedReader;
import java.util.Hashtable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
public class PreprocessorContext {
private Hashtable<String , String> param_val ;
@@ -210,7 +210,8 @@ public class PreprocessorContext {
return streamData.trim();
}
- private Pattern id_pattern = Pattern.compile("\\$[_]*[a-zA-Z][a-zA-Z_0-9]*");
+ private Pattern bracketIdPattern = Pattern.compile("\\$\\{([_]*[a-zA-Z][a-zA-Z_0-9]*)\\}");
+ private Pattern id_pattern = Pattern.compile("\\$([_]*[a-zA-Z][a-zA-Z_0-9]*)");
public String substitute(String line) {
@@ -219,15 +220,35 @@ public class PreprocessorContext {
String replaced_line = line;
- Matcher keyMatcher = id_pattern.matcher( line );
+ Matcher bracketKeyMatcher = bracketIdPattern.matcher(line);
+
String key="";
String val="";
+ while (bracketKeyMatcher.find()) {
+ if ( (bracketKeyMatcher.start() == 0) || (line.charAt( bracketKeyMatcher.start() - 1)) != '\\' ) {
+ key = bracketKeyMatcher.group(1);
+ if (!(param_val.containsKey(key))) {
+ throw new RuntimeException("Undefined parameter : "+key);
+ }
+ val = param_val.get(key);
+ if (val.contains("$")) {
+ val = val.replaceAll("(?<!\\\\)\\$", "\\\\\\$");
+ }
+ replaced_line = replaced_line.replaceFirst("\\$\\{"+key+"\\}", val);
+ }
+ }
+
+ Matcher keyMatcher = id_pattern.matcher( replaced_line );
+
+ key="";
+ val="";
+
while (keyMatcher.find()) {
// make sure that we don't perform parameter substitution
// for escaped vars of the form \$<id>
if ( (keyMatcher.start() == 0) || (line.charAt( keyMatcher.start() - 1)) != '\\' ) {
- key = keyMatcher.group().substring(1); //skip the '$'
+ key = keyMatcher.group(1);
if (!(param_val.containsKey(key))) {
throw new RuntimeException("Undefined parameter : "+key);
}
Modified: pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1434521&r1=1434520&r2=1434521&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Thu Jan 17 00:27:22 2013
@@ -1304,4 +1304,39 @@ public class TestParamSubPreproc {
log.info("Done");
}
+
+ @Test
+ public void testCmdlineParamCurlySyntax() throws Exception{
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir + "/input7.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"date=20080228"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult7.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
+ }
}
\ No newline at end of file
Added: pig/trunk/test/org/apache/pig/test/data/ExpectedResult7.pig
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/ExpectedResult7.pig?rev=1434521&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/ExpectedResult7.pig (added)
+++ pig/trunk/test/org/apache/pig/test/data/ExpectedResult7.pig Thu Jan 17 00:27:22 2013
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228_woah' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: pig/trunk/test/org/apache/pig/test/data/input7.pig
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/input7.pig?rev=1434521&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/input7.pig (added)
+++ pig/trunk/test/org/apache/pig/test/data/input7.pig Thu Jan 17 00:27:22 2013
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/${date}_woah' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';