You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2012/01/06 23:07:04 UTC
svn commit: r1228461 - in /pig/trunk: ./
src/org/apache/pig/newplan/logical/expression/
src/org/apache/pig/newplan/logical/visitor/ src/org/apache/pig/parser/
test/org/apache/pig/test/
Author: thejas
Date: Fri Jan 6 22:07:03 2012
New Revision: 1228461
URL: http://svn.apache.org/viewvc?rev=1228461&view=rev
Log:
PIG-2430: An EvalFunc which overrides getArgToFuncMapping with FuncSpec
with constructor arguments is not properly instantiated with said arguments (jcoveney via thejas)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
pig/trunk/test/org/apache/pig/test/TestUDF.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1228461&r1=1228460&r2=1228461&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Jan 6 22:07:03 2012
@@ -210,6 +210,9 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2430: An EvalFunc which overrides getArgToFuncMapping with FuncSpec
+ with constructor arguments is not properly instantiated with said arguments (jcoveney via thejas)
+
PIG-2457: JsonLoaderStorage tests is broken for e2e (daijy)
PIG-2426: ProgressableReporter.progress(String msg) is an empty function (vivekp via daijy)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java?rev=1228461&r1=1228460&r2=1228461&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java Fri Jan 6 22:07:03 2012
@@ -45,6 +45,7 @@ public class UserFuncExpression extends
private EvalFunc<?> ef = null;
private String signature;
private static int sigSeq=0;
+ private boolean viaDefine=false; //this represents whether the function was instantiate via a DEFINE statement or not
public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec) {
super("UserFunc", plan);
@@ -64,6 +65,16 @@ public class UserFuncExpression extends
}
}
+ public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec, boolean viaDefine) {
+ this( plan, funcSpec);
+ this.viaDefine = viaDefine;
+ }
+
+ public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec, List<LogicalExpression> args, boolean viaDefine) {
+ this( plan, funcSpec, args );
+ this.viaDefine = viaDefine;
+ }
+
public FuncSpec getFuncSpec() {
return mFuncSpec;
}
@@ -213,7 +224,8 @@ public class UserFuncExpression extends
try {
copy = new UserFuncExpression(
lgExpPlan,
- this.getFuncSpec().clone() );
+ this.getFuncSpec().clone(),
+ viaDefine);
copy.signature = signature;
// Deep copy the input expressions.
@@ -255,4 +267,7 @@ public class UserFuncExpression extends
return signature;
}
+ public boolean isViaDefine() {
+ return viaDefine;
+ }
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java?rev=1228461&r1=1228460&r2=1228461&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java Fri Jan 6 22:07:03 2012
@@ -784,7 +784,9 @@ public class TypeCheckingExpVisitor exte
"different input argument types, please use explicit casts.";
msgCollector.collect(msg, MessageType.Warning, PigWarning.USING_OVERLOADED_FUNCTION);
}
+ if (func.isViaDefine()) {
matchingSpec.setCtorArgs(func.getFuncSpec().getCtorArgs());
+ }
func.setFuncSpec(matchingSpec);
insertCastsForUDF(func, currentArgSchema, matchingSpec.getInputArgsSchema());
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1228461&r1=1228460&r2=1228461&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Fri Jan 6 22:07:03 2012
@@ -981,12 +981,17 @@ public class LogicalPlanBuilder {
}
FuncSpec funcSpec = pigContext.getFuncSpecFromAlias( funcName );
+ LogicalExpression le;
if( funcSpec == null ) {
funcName = func.getClass().getName();
funcSpec = new FuncSpec( funcName );
+ //this point is only reached if there was no DEFINE statement for funcName
+ //in which case, we pass that information along
+ le = new UserFuncExpression( plan, funcSpec, args, false );
+ } else {
+ le = new UserFuncExpression( plan, funcSpec, args, true );
}
- LogicalExpression le = new UserFuncExpression( plan, funcSpec, args );
le.setLocation( loc );
return le;
}
Modified: pig/trunk/test/org/apache/pig/test/TestUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestUDF.java?rev=1228461&r1=1228460&r2=1228461&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestUDF.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestUDF.java Fri Jan 6 22:07:03 2012
@@ -23,14 +23,19 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
import junit.framework.TestCase;
import org.apache.pig.ExecType;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
import org.apache.pig.PigServer;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.test.utils.MyUDFReturnMap;
import org.junit.After;
@@ -54,12 +59,14 @@ public class TestUDF extends TestCase {
@Override
@Before
public void setUp() throws Exception {
+ FileLocalizer.setInitialized(false);
TempScriptFile = File.createTempFile("temp_jira_851", ".pig");
FileWriter writer = new FileWriter(TempScriptFile);
for (String line : ScriptStatement) {
writer.write(line + "\n");
}
writer.close();
+
}
@AfterClass
@@ -133,9 +140,85 @@ public class TestUDF extends TestCase {
}
}
+ //see PIG-2430
+ @Test
+ public void testEvalFuncGetArgToFunc() throws Throwable {
+ String input = "udf_test_jira_2430.txt";
+ Util.createLocalInputFile(input, new String[]{"1,hey"});
+ try {
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ pigServer.registerQuery("A = LOAD '"+input+"' USING PigStorage(',') AS (x:int,y:chararray);");
+ pigServer.registerQuery("B = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs(x);");
+ pigServer.registerQuery("C = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs(y);");
+ pigServer.registerQuery("D = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs((long)x);");
+ pigServer.registerQuery("E = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs((double)x);");
+ pigServer.registerQuery("F = FOREACH A GENERATE org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs((float)x);");
+ Iterator<Tuple> it = pigServer.openIterator("B");
+ assertEquals(Integer.valueOf(2),(Integer)it.next().get(0));
+ it = pigServer.openIterator("C");
+ assertEquals(Integer.valueOf(1), (Integer)it.next().get(0));
+ it = pigServer.openIterator("D");
+ assertEquals(Integer.valueOf(3),(Integer)it.next().get(0));
+ it = pigServer.openIterator("E");
+ assertEquals(Integer.valueOf(4),(Integer)it.next().get(0));
+ it = pigServer.openIterator("F");
+ assertEquals(Integer.valueOf(5),(Integer)it.next().get(0));
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ //see PIG-2430
+ @Test
+ public void testNormalDefine() throws Throwable {
+ String input = "udf_test_jira_2430_2.txt";
+ Util.createLocalInputFile(input, new String[]{"1"});
+ try {
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ pigServer.registerQuery("A = LOAD '"+input+"' as (x:int);");
+ pigServer.registerQuery("DEFINE udftest1 org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs('1');");
+ pigServer.registerQuery("DEFINE udftest2 org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs('2');");
+ pigServer.registerQuery("DEFINE udftest3 org.apache.pig.test.TestUDF$UdfWithFuncSpecWithArgs('3');");
+ pigServer.registerQuery("B = FOREACH A GENERATE udftest1(x), udftest2(x), udftest3(x);");
+ Iterator<Tuple> its = pigServer.openIterator("B");
+ Tuple t = its.next();
+ assertEquals(Integer.valueOf(1),t.get(0));
+ assertEquals(Integer.valueOf(2),t.get(1));
+ assertEquals(Integer.valueOf(3),t.get(2));
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
@Override
@After
public void tearDown() throws Exception {
TempScriptFile.delete();
}
+
+ public static class UdfWithFuncSpecWithArgs extends EvalFunc<Integer> {
+ private Integer ret = 0;
+ public UdfWithFuncSpecWithArgs() {}
+ public UdfWithFuncSpecWithArgs(String ret) {
+ this.ret=Integer.parseInt(ret);
+ }
+
+ @Override
+ public Integer exec(Tuple input) throws IOException {
+ return ret;
+ }
+
+ @Override
+ public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+ List<FuncSpec> l = new ArrayList<FuncSpec>(5);
+ l.add(new FuncSpec(this.getClass().getName(), new String[]{"1"}, new Schema(new Schema.FieldSchema(null,DataType.CHARARRAY))));
+ l.add(new FuncSpec(this.getClass().getName(), new String[]{"2"}, new Schema(new Schema.FieldSchema(null,DataType.INTEGER))));
+ l.add(new FuncSpec(this.getClass().getName(), new String[]{"3"}, new Schema(new Schema.FieldSchema(null,DataType.LONG))));
+ l.add(new FuncSpec(this.getClass().getName(), new String[]{"4"}, new Schema(new Schema.FieldSchema(null,DataType.DOUBLE))));
+ l.add(new FuncSpec(this.getClass().getName(), new String[]{"5"}, new Schema(new Schema.FieldSchema(null,DataType.FLOAT))));
+ return l;
+ }
+ }
}