You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/04/24 18:28:21 UTC
svn commit: r651307 - in /incubator/pig/trunk: CHANGES.txt
src/org/apache/pig/tools/parameters/PreprocessorContext.java
test/org/apache/pig/test/TestParamSubPreproc.java
test/org/apache/pig/test/data/output1.pig
Author: olga
Date: Thu Apr 24 09:28:12 2008
New Revision: 651307
URL: http://svn.apache.org/viewvc?rev=651307&view=rev
Log:
PIG-218: fixed parameter generation code to work with arbitrary commands
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig
Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=651307&r1=651306&r2=651307&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Thu Apr 24 09:28:12 2008
@@ -244,9 +244,11 @@
PIG-213: Remove non-static references to logger from data bags and tuples,
as it causes significant overhead (vgeschel via gates).
- PIG-216 Fix streaming to work with commands that use unix pipes (acmurthy
+ PIG-216: Fix streaming to work with commands that use unix pipes (acmurthy
via gates).
- PIG-207 Fix illustrate command to work in mapreduce mode (shubhamc via
+ PIG-207: Fix illustrate command to work in mapreduce mode (shubhamc via
gates).
+
+ PIG-218: Fixed param generation to work with arbitrary commands
Modified: incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=651307&r1=651306&r2=651307&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Thu Apr 24 09:28:12 2008
@@ -142,7 +142,16 @@
String streamError="";
try {
log.info("Executing command : " + cmd);
- p = Runtime.getRuntime().exec(cmd,null);
+ // we can't use exec directly since it does not handle
+ // case like foo -c "bar bar" correctly. It splits on white spaces even in presents of quotes
+ String[] cmdArgs = new String[3];
+ cmdArgs[0] = "bash";
+ cmdArgs[1] = "-c";
+ StringBuffer sb = new StringBuffer("exec ");
+ sb.append(cmd);
+ cmdArgs[2] = sb.toString();
+
+ p = Runtime.getRuntime().exec(cmdArgs);
} catch (IOException e) {
RuntimeException rte = new RuntimeException("IO Exception while executing shell command : "+e.getMessage() , e);
Modified: incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=651307&r1=651306&r2=651307&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Thu Apr 24 09:28:12 2008
@@ -1185,4 +1185,53 @@
}
+ /* Test case 31
+ * Use of inline command
+ */
+ @Test
+ public void testCmdlineParamWithInlineCmd() throws Exception{
+ log.info("Starting test testCmdlineParam() ...");
+ try {
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"date=`/usr/local/bin/perl -e 'print \"20080228\n20070101\"' | head -n 1`"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
+ } catch (ParseException e) {
+ fail ("Got ParseException : " + e.getMessage());
+ } catch (RuntimeException e) {
+ fail ("Got RuntimeException : " + e.getMessage());
+ } catch (Error e) {
+ fail ("Got error : " + e.getMessage());
+ }
+
+ log.info("Done");
+
+ }
}
Modified: incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=651307&r1=651306&r2=651307&view=diff
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig (original)
+++ incubator/pig/trunk/test/org/apache/pig/test/data/output1.pig Thu Apr 24 09:28:12 2008
@@ -4,7 +4,7 @@
a = foreach bb generate $0,$12,$7;
--generate inactive accts
-inactiveAccounts = filter a by ($1 neq '$column1') and ($1 == '2') parallel 400;
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
grpInactiveAcct = group inactiveAccounts all;
countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }