You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/04/13 00:23:38 UTC
svn commit: r1091585 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PONative.java
src/org/apache/pig/parser/StreamingCommandUtils.java
test/org/apache/pig/test/TestNativeMapReduce.java
Author: thejas
Date: Tue Apr 12 22:23:38 2011
New Revision: 1091585
URL: http://svn.apache.org/viewvc?rev=1091585&view=rev
Log:
PIG-1917: NativeMapReduce does not Allow Configuration Parameters containing Spaces (thejas)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PONative.java
pig/trunk/src/org/apache/pig/parser/StreamingCommandUtils.java
pig/trunk/test/org/apache/pig/test/TestNativeMapReduce.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1091585&r1=1091584&r2=1091585&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Apr 12 22:23:38 2011
@@ -142,6 +142,9 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1917: NativeMapReduce does not Allow Configuration Parameters
+ containing Spaces (thejas)
+
PIG-1974: Lineage need to set for every cast (thejas)
PIG-1988: Importing an empty macro file causing NPE (rding)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PONative.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PONative.java?rev=1091585&r1=1091584&r2=1091585&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PONative.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PONative.java Tue Apr 12 22:23:38 2011
@@ -19,10 +19,10 @@ package org.apache.pig.backend.hadoop.ex
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.Utils;
-import org.apache.pig.data.Tuple;
public class PONative extends PhysicalOperator {
@@ -43,7 +43,7 @@ public class PONative extends PhysicalOp
@Override
public String name() {
return getAliasString() + "Native" + "('hadoop jar "
- + nativeMRjar + " " + Utils.getStringFromArray(params) + "')"
+ + nativeMRjar + " " + Utils.getStringFromArray(getParams()) + "')"
+ " - " + mKey.toString();
}
@@ -56,9 +56,38 @@ public class PONative extends PhysicalOp
}
public String[] getParams() {
+ unquotePropertyParams();
return params;
}
+ /**
+ * if there is a argument that starts with "-D", unquote the value part
+ * to support use case in PIG-1917
+ */
+ private void unquotePropertyParams() {
+ for(int i=0; i<params.length; i++){
+ String param = params[i];
+ if(param.startsWith("-D")){
+ int equalPos = param.indexOf('=');
+ //to unquote, there should be a '=', then at least two quotes
+ if(equalPos == -1 || equalPos >= param.length() - 3)
+ continue;
+
+ if(checkQuote(equalPos+1, param,'\'')
+ || checkQuote(equalPos + 1, param, '"')
+ ){
+ //found quoted value part, remove the quotes
+ params[i] = param.substring(0, equalPos + 1)
+ + param.substring(equalPos + 2, param.length() - 1);
+ }
+ }
+ }
+ }
+
+ private boolean checkQuote(int i, String param, char quote) {
+ return param.charAt(i) == quote && param.charAt(param.length()-1) == quote;
+ }
+
public void setParams(String[] params) {
this.params = params;
}
Modified: pig/trunk/src/org/apache/pig/parser/StreamingCommandUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/StreamingCommandUtils.java?rev=1091585&r1=1091584&r2=1091585&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/StreamingCommandUtils.java (original)
+++ pig/trunk/src/org/apache/pig/parser/StreamingCommandUtils.java Tue Apr 12 22:23:38 2011
@@ -54,8 +54,11 @@ public class StreamingCommandUtils {
List<String> argv = new ArrayList<String>();
int beginIndex = 0;
+ int endIndex = -1;
+ for( ; beginIndex < command.length(); beginIndex = endIndex + 1){
+ // look for next arg in string
+ String arg = "";
- while (beginIndex < command.length()) {
// Skip spaces
while (Character.isWhitespace(command.charAt(beginIndex))) {
++beginIndex;
@@ -63,30 +66,45 @@ public class StreamingCommandUtils {
char delim = ' ';
char charAtIndex = command.charAt(beginIndex);
+
+ //find the end of this arg
+ endIndex = beginIndex + 1;
if (charAtIndex == SINGLE_QUOTE || charAtIndex == DOUBLE_QUOTE) {
delim = charAtIndex;
}
-
- int endIndex = command.indexOf(delim, beginIndex+1);
- if (endIndex == -1) {
- if (Character.isWhitespace(delim)) {
- // Reached end of command-line
- argv.add(command.substring(beginIndex));
- break;
- } else {
- // Didn't find the ending quote/double-quote
- throw new ParseException("Illegal command: " + command);
+ else{
+ //space delim
+ while(endIndex < command.length()){
+ char charAtEndIdx = command.charAt(endIndex);
+ if(charAtEndIdx == ' '){
+ // found the next space delim
+ break;
+ }else if(charAtEndIdx == SINGLE_QUOTE || charAtEndIdx == DOUBLE_QUOTE){
+ //switch to new delim so that strings like
+ // -Dprop='abc xyz' are parsed as one arg
+ arg = command.substring(beginIndex, endIndex);
+ beginIndex = endIndex;
+ endIndex = beginIndex + 1;
+ delim = charAtEndIdx;
+ break;
+ }
+ endIndex++;
+ }
+ if(delim == ' '){
+ // reached end of string or next space
+ argv.add(command.substring(beginIndex, endIndex));
+ continue;
}
}
- if (Character.isWhitespace(delim)) {
- // Do not consume the space
- argv.add(command.substring(beginIndex, endIndex));
- } else {
- argv.add(command.substring(beginIndex, endIndex+1));
+ //one of the quote delims
+ endIndex = command.indexOf(delim, endIndex);
+ if (endIndex == -1) {
+ // Didn't find the ending quote/double-quote
+ throw new ParseException("Illegal command: " + command);
}
+ argv.add(arg + command.substring(beginIndex, endIndex+1));
- beginIndex = endIndex + 1;
}
return argv.toArray(new String[argv.size()]);
Modified: pig/trunk/test/org/apache/pig/test/TestNativeMapReduce.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNativeMapReduce.java?rev=1091585&r1=1091584&r2=1091585&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNativeMapReduce.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNativeMapReduce.java Tue Apr 12 22:23:38 2011
@@ -128,7 +128,9 @@ public class TestNativeMapReduce {
pigServer.registerQuery("B = mapreduce '" + jarFileName + "' " +
"Store A into 'table_testNativeMRJobSimple_input' "+
"Load 'table_testNativeMRJobSimple_output' "+
- "`org.apache.pig.test.utils.WordCount -files " + STOPWORD_FILE +
+ "`org.apache.pig.test.utils.WordCount " +
+ " -Dmapred.child.java.opts='-Xmx1536m -Xms128m' " +
+ " -files " + STOPWORD_FILE +
" table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output " +
STOPWORD_FILE + "`;");
pigServer.registerQuery("Store B into 'table_testNativeMRJobSimpleDir';");