You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/11/02 22:44:50 UTC
svn commit: r1196816 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/
src/org/apache/pig/newplan/logical/expression/ test/e2e/pig/tests/
test/e2e/pig/udfs/java/org/apache/pig/t...
Author: daijy
Date: Wed Nov 2 21:44:49 2011
New Revision: 1196816
URL: http://svn.apache.org/viewvc?rev=1196816&view=rev
Log:
PIG-2338: Need signature for EvalFunc
Added:
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/EvalFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Nov 2 21:44:49 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2338: Need signature for EvalFunc (daijy)
+
OPTIMIZATIONS
BUG FIXES
Modified: pig/trunk/src/org/apache/pig/EvalFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/EvalFunc.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/EvalFunc.java (original)
+++ pig/trunk/src/org/apache/pig/EvalFunc.java Wed Nov 2 21:44:49 2011
@@ -33,6 +33,8 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.LoadPushDown.RequiredFieldList;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigLogger;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigProgressable;
@@ -69,7 +71,7 @@ public abstract class EvalFunc<T> {
* should be logged to this via {@link PigLogger#warn}.
*/
protected PigLogger pigLogger;
-
+
private static int nextSchemaId; // for assigning unique ids to UDF columns
protected String getSchemaName(String name, Schema input) {
String alias = name + "_";
@@ -293,4 +295,15 @@ public abstract class EvalFunc<T> {
public Log getLogger() {
return log;
}
+
+ /**
+ * This method will be called by Pig both in the front end and back end to
+ * pass a unique signature to the {@link EvalFunc}. The signature can be used
+ * to store into the {@link UDFContext} any information which the
+ * {@link EvalFunc} needs to store between various method invocations in the
+ * front end and back end.
+ * @param signature a unique signature to identify this EvalFunc
+ */
+ public void setUDFContextSignature(String signature) {
+ }
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java Wed Nov 2 21:44:49 2011
@@ -23,12 +23,14 @@ import java.io.ObjectInputStream;
import java.lang.reflect.Type;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import org.apache.pig.Accumulator;
import org.apache.pig.Algebraic;
import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
+import org.apache.pig.ResourceSchema;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
@@ -46,6 +48,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.util.UDFContext;
public class POUserFunc extends ExpressionOperator {
@@ -66,6 +69,7 @@ public class POUserFunc extends Expressi
private PhysicalOperator referencedOperator = null;
private boolean isAccumulationDone;
+ private String signature;
public PhysicalOperator getReferencedOperator() {
return referencedOperator;
@@ -105,6 +109,7 @@ public class POUserFunc extends Expressi
private void instantiateFunc(FuncSpec fSpec) {
this.func = (EvalFunc) PigContext.instantiateFuncFromSpec(fSpec);
+ this.func.setUDFContextSignature(signature);
if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
executor = new MonitoredUDFExecutor(func);
}
@@ -431,6 +436,7 @@ public class POUserFunc extends Expressi
NodeIdGenerator.getGenerator().getNextNodeId(mKey.scope)),
requestedParallelism, null, funcSpec.clone());
clone.setResultType(resultType);
+ clone.signature = signature;
return clone;
}
@@ -469,4 +475,11 @@ public class POUserFunc extends Expressi
public EvalFunc getFunc() {
return func;
}
+
+ public void setSignature(String signature) {
+ this.signature = signature;
+ if (this.func!=null) {
+ this.func.setUDFContextSignature(signature);
+ }
+ }
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java Wed Nov 2 21:44:49 2011
@@ -505,6 +505,7 @@ public class ExpToPhyTranslationVisitor
p = new POUserFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
.getNextNodeId(DEFAULT_SCOPE)), -1,
null, op.getFuncSpec(), (EvalFunc) f);
+ ((POUserFunc)p).setSignature(op.getSignature());
List<String> cacheFiles = ((EvalFunc)f).getCacheFiles();
if (cacheFiles != null) {
((POUserFunc)p).setCacheFiles(cacheFiles.toArray(new String[cacheFiles.size()]));
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java Wed Nov 2 21:44:49 2011
@@ -21,13 +21,17 @@ package org.apache.pig.newplan.logical.e
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Properties;
+
import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
+import org.apache.pig.ResourceSchema;
import org.apache.pig.builtin.Nondeterministic;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.PlanVisitor;
@@ -39,11 +43,16 @@ public class UserFuncExpression extends
private FuncSpec mFuncSpec;
private EvalFunc<?> ef = null;
+ private String signature;
+ private static int sigSeq=0;
public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec) {
super("UserFunc", plan);
mFuncSpec = funcSpec;
plan.add(this);
+ if (signature==null) {
+ signature = Integer.toString(sigSeq++);
+ }
}
@@ -170,6 +179,8 @@ public class UserFuncExpression extends
// This significantly optimize the performance of frontend (PIG-1738)
if (ef==null)
ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
+
+ ef.setUDFContextSignature(signature);
Schema udfSchema = ef.outputSchema(Util.translateSchema(inputSchema));
if (udfSchema != null) {
@@ -198,6 +209,7 @@ public class UserFuncExpression extends
lgExpPlan,
this.getFuncSpec().clone() );
+ copy.signature = signature;
// Deep copy the input expressions.
List<Operator> inputs = plan.getSuccessors( this );
if( inputs != null ) {
@@ -232,5 +244,9 @@ public class UserFuncExpression extends
return msg.toString();
}
+
+ public String getSignature() {
+ return signature;
+ }
}
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1196816&r1=1196815&r2=1196816&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Wed Nov 2 21:44:49 2011
@@ -3882,6 +3882,27 @@ store E into ':OUTPATH:';\,
store D into ':OUTPATH:';?,
}
],
+ },{
+ 'name' => 'UDFContext',
+ 'tests' => [
+ {
+ # See PIG-2338
+ 'num' => 1,
+ 'pig' => q?register :FUNCPATH:/testudf.jar
+ a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
+ b = foreach a generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(a0);
+ c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
+ d = foreach c generate org.apache.pig.test.udf.evalfunc.UDFContextTestUDF(c0);
+ e = union b, d;
+ store e into ':OUTPATH:';?,
+ 'verify_pig_script' => q?a = load ':INPATH:/singlefile/studenttab10k' AS (a0);
+ b = foreach a generate '{a0: bytearray}';
+ c = load ':INPATH:/singlefile/studenttab10k' AS (c0:chararray);
+ d = foreach c generate '{c0: chararray}';
+ e = union b, d;
+ store e into ':OUTPATH:';?,
+ }
+ ],
},
],
},
Added: pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java?rev=1196816&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java (added)
+++ pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UDFContextTestUDF.java Wed Nov 2 21:44:49 2011
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.udf.evalfunc;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.UDFContext;
+
+public class UDFContextTestUDF extends EvalFunc<String> {
+ String signature;
+
+ @Override
+ public String exec(Tuple input) throws IOException {
+ Schema sch = (Schema)UDFContext.getUDFContext()
+ .getUDFProperties(this.getClass()).get("pig.evalfunc.signature."+signature);
+ return sch.toString();
+ }
+
+ @Override
+ public void setUDFContextSignature(String signature) {
+ this.signature = signature;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ Properties props = UDFContext.getUDFContext().getUDFProperties(this.getClass());
+ props.put("pig.evalfunc.signature."+signature, input);
+ return new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY));
+ }
+}