You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2011/02/25 04:44:51 UTC
svn commit: r1074393 - in /hive/trunk:
contrib/src/java/org/apache/hadoop/hive/contrib/udf/
contrib/src/test/queries/clientnegative/
contrib/src/test/queries/clientpositive/
contrib/src/test/results/clientnegative/
contrib/src/test/results/clientpositi...
Author: cws
Date: Fri Feb 25 03:44:50 2011
New Revision: 1074393
URL: http://svn.apache.org/viewvc?rev=1074393&view=rev
Log:
HIVE-1994 Support new annotation @UDFType(stateful = true) (John Sichi via cws)
Added:
hive/trunk/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
hive/trunk/contrib/src/test/queries/clientnegative/invalid_row_sequence.q
hive/trunk/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out
hive/trunk/contrib/src/test/results/clientnegative/invalid_row_sequence.q.out
Modified:
hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udf/UDFRowSequence.java
hive/trunk/contrib/src/test/queries/clientpositive/udf_row_sequence.q
hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFType.java
hive/trunk/ql/src/test/queries/clientpositive/udf_case.q
hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out
Modified: hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udf/UDFRowSequence.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udf/UDFRowSequence.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udf/UDFRowSequence.java (original)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udf/UDFRowSequence.java Fri Feb 25 03:44:50 2011
@@ -28,7 +28,7 @@ import org.apache.hadoop.io.LongWritable
*/
@Description(name = "row_sequence",
value = "_FUNC_() - Returns a generated row sequence number starting from 1")
-@UDFType(deterministic = false)
+@UDFType(deterministic = false, stateful = true)
public class UDFRowSequence extends UDF
{
private LongWritable result = new LongWritable();
Added: hive/trunk/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/queries/clientnegative/case_with_row_sequence.q?rev=1074393&view=auto
==============================================================================
--- hive/trunk/contrib/src/test/queries/clientnegative/case_with_row_sequence.q (added)
+++ hive/trunk/contrib/src/test/queries/clientnegative/case_with_row_sequence.q Fri Feb 25 03:44:50 2011
@@ -0,0 +1,10 @@
+drop temporary function row_sequence;
+
+add jar ${system:build.dir}/hive-contrib-${system:hive.version}.jar;
+create temporary function row_sequence as
+'org.apache.hadoop.hive.contrib.udf.UDFRowSequence';
+
+-- make sure a stateful function inside of CASE throws an exception
+-- since the short-circuiting requirements are contradictory
+SELECT CASE WHEN 3 > 2 THEN 10 WHEN row_sequence() > 5 THEN 20 ELSE 30 END
+FROM src LIMIT 1;
Added: hive/trunk/contrib/src/test/queries/clientnegative/invalid_row_sequence.q
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/queries/clientnegative/invalid_row_sequence.q?rev=1074393&view=auto
==============================================================================
--- hive/trunk/contrib/src/test/queries/clientnegative/invalid_row_sequence.q (added)
+++ hive/trunk/contrib/src/test/queries/clientnegative/invalid_row_sequence.q Fri Feb 25 03:44:50 2011
@@ -0,0 +1,13 @@
+-- Verify that a stateful UDF cannot be used outside of the SELECT list
+
+drop temporary function row_sequence;
+
+add jar ${system:build.dir}/hive-contrib-${system:hive.version}.jar;
+
+create temporary function row_sequence as
+'org.apache.hadoop.hive.contrib.udf.UDFRowSequence';
+
+select key
+from (select key from src order by key) x
+where row_sequence() < 5
+order by key;
Modified: hive/trunk/contrib/src/test/queries/clientpositive/udf_row_sequence.q
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/queries/clientpositive/udf_row_sequence.q?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/queries/clientpositive/udf_row_sequence.q (original)
+++ hive/trunk/contrib/src/test/queries/clientpositive/udf_row_sequence.q Fri Feb 25 03:44:50 2011
@@ -22,4 +22,10 @@ select key, row_sequence() as r
from (select key from src order by key) x
order by r;
+-- make sure stateful functions do not get short-circuited away
+-- a true result for key=105 would indicate undesired short-circuiting
+select key, (key = 105) and (row_sequence() = 1)
+from (select key from src order by key) x
+order by key limit 20;
+
drop temporary function row_sequence;
Added: hive/trunk/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out?rev=1074393&view=auto
==============================================================================
--- hive/trunk/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out (added)
+++ hive/trunk/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out Fri Feb 25 03:44:50 2011
@@ -0,0 +1,18 @@
+PREHOOK: query: drop temporary function row_sequence
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: drop temporary function row_sequence
+POSTHOOK: type: DROPFUNCTION
+PREHOOK: query: create temporary function row_sequence as
+'org.apache.hadoop.hive.contrib.udf.UDFRowSequence'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: create temporary function row_sequence as
+'org.apache.hadoop.hive.contrib.udf.UDFRowSequence'
+POSTHOOK: type: CREATEFUNCTION
+PREHOOK: query: -- make sure a stateful function inside of CASE throws an exception
+-- since the short-circuiting requirements are contradictory
+SELECT CASE WHEN 3 > 2 THEN 10 WHEN row_sequence() > 5 THEN 20 ELSE 30 END
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_23-14-39_576_5692614556807208481/-mr-10000
+FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask
Added: hive/trunk/contrib/src/test/results/clientnegative/invalid_row_sequence.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientnegative/invalid_row_sequence.q.out?rev=1074393&view=auto
==============================================================================
--- hive/trunk/contrib/src/test/results/clientnegative/invalid_row_sequence.q.out (added)
+++ hive/trunk/contrib/src/test/results/clientnegative/invalid_row_sequence.q.out Fri Feb 25 03:44:50 2011
@@ -0,0 +1,15 @@
+PREHOOK: query: -- Verify that a stateful UDF cannot be used outside of the SELECT list
+
+drop temporary function row_sequence
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: -- Verify that a stateful UDF cannot be used outside of the SELECT list
+
+drop temporary function row_sequence
+POSTHOOK: type: DROPFUNCTION
+PREHOOK: query: create temporary function row_sequence as
+'org.apache.hadoop.hive.contrib.udf.UDFRowSequence'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: create temporary function row_sequence as
+'org.apache.hadoop.hive.contrib.udf.UDFRowSequence'
+POSTHOOK: type: CREATEFUNCTION
+FAILED: Error in semantic analysis: Stateful UDF's can only be invoked in the SELECT list
Modified: hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out Fri Feb 25 03:44:50 2011
@@ -79,7 +79,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-16_20-11-16_760_3970553477134855753/-mr-10002
+ file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-26-27_614_6714509994567553903/-mr-10002
Reduce Output Operator
key expressions:
expr: _col1
@@ -110,13 +110,13 @@ from (select key from src order by key)
order by r
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-16_20-11-16_840_7492426874452808984/-mr-10000
+PREHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-26-27_973_4395636849347592535/-mr-10000
POSTHOOK: query: select key, row_sequence() as r
from (select key from src order by key) x
order by r
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-16_20-11-16_840_7492426874452808984/-mr-10000
+POSTHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-26-27_973_4395636849347592535/-mr-10000
0 1
0 2
0 3
@@ -617,6 +617,42 @@ POSTHOOK: Output: file:/tmp/sdong/hive_2
97 498
98 499
98 500
+PREHOOK: query: -- make sure stateful functions do not get short-circuited away
+-- a true result for key=105 would indicate undesired short-circuiting
+select key, (key = 105) and (row_sequence() = 1)
+from (select key from src order by key) x
+order by key limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-26-43_815_3305291736093408641/-mr-10000
+POSTHOOK: query: -- make sure stateful functions do not get short-circuited away
+-- a true result for key=105 would indicate undesired short-circuiting
+select key, (key = 105) and (row_sequence() = 1)
+from (select key from src order by key) x
+order by key limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-26-43_815_3305291736093408641/-mr-10000
+0 false
+0 false
+0 false
+10 false
+100 false
+100 false
+103 false
+103 false
+104 false
+104 false
+105 false
+11 false
+111 false
+113 false
+113 false
+114 false
+116 false
+118 false
+118 false
+119 false
PREHOOK: query: drop temporary function row_sequence
PREHOOK: type: DROPFUNCTION
POSTHOOK: query: drop temporary function row_sequence
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java Fri Feb 25 03:44:50 2011
@@ -21,8 +21,11 @@ package org.apache.hadoop.hive.ql.exec;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
@@ -39,7 +42,8 @@ public class ExprNodeGenericFuncEvaluato
transient GenericUDF genericUDF;
transient Object rowObject;
transient ExprNodeEvaluator[] children;
- transient DeferredExprObject[] deferredChildren;
+ transient GenericUDF.DeferredObject[] deferredChildren;
+ transient boolean isEager;
/**
* Class to allow deferred evaluation for GenericUDF.
@@ -55,17 +59,60 @@ public class ExprNodeGenericFuncEvaluato
public Object get() throws HiveException {
return eval.evaluate(rowObject);
}
- };
+ }
+
+ /**
+ * Class to force eager evaluation for GenericUDF in cases where
+ * it is warranted.
+ */
+ class EagerExprObject implements GenericUDF.DeferredObject {
+
+ ExprNodeEvaluator eval;
+
+ transient Object obj;
+
+ EagerExprObject(ExprNodeEvaluator eval) {
+ this.eval = eval;
+ }
+ void evaluate() throws HiveException {
+ obj = eval.evaluate(rowObject);
+ }
+
+ public Object get() throws HiveException {
+ return obj;
+ }
+ }
+
public ExprNodeGenericFuncEvaluator(ExprNodeGenericFuncDesc expr) {
this.expr = expr;
children = new ExprNodeEvaluator[expr.getChildExprs().size()];
+ isEager = false;
for (int i = 0; i < children.length; i++) {
- children[i] = ExprNodeEvaluatorFactory.get(expr.getChildExprs().get(i));
+ ExprNodeDesc child = expr.getChildExprs().get(i);
+ ExprNodeEvaluator nodeEvaluator = ExprNodeEvaluatorFactory.get(child);
+ children[i] = nodeEvaluator;
+ // If we have eager evaluators anywhere below us, then we are eager too.
+ if (nodeEvaluator instanceof ExprNodeGenericFuncEvaluator) {
+ if (((ExprNodeGenericFuncEvaluator) nodeEvaluator).isEager) {
+ isEager = true;
+ }
+ // Base case: we are eager if a child is stateful
+ GenericUDF childUDF =
+ ((ExprNodeGenericFuncDesc) child).getGenericUDF();
+ if (FunctionRegistry.isStateful(childUDF)) {
+ isEager = true;
+ }
+ }
}
- deferredChildren = new DeferredExprObject[expr.getChildExprs().size()];
+ deferredChildren =
+ new GenericUDF.DeferredObject[expr.getChildExprs().size()];
for (int i = 0; i < deferredChildren.length; i++) {
- deferredChildren[i] = new DeferredExprObject(children[i]);
+ if (isEager) {
+ deferredChildren[i] = new EagerExprObject(children[i]);
+ } else {
+ deferredChildren[i] = new DeferredExprObject(children[i]);
+ }
}
}
@@ -77,12 +124,23 @@ public class ExprNodeGenericFuncEvaluato
childrenOIs[i] = children[i].initialize(rowInspector);
}
genericUDF = expr.getGenericUDF();
+ if (isEager &&
+ ((genericUDF instanceof GenericUDFCase)
+ || (genericUDF instanceof GenericUDFWhen))) {
+ throw new HiveException(
+ "Stateful expressions cannot be used inside of CASE");
+ }
return genericUDF.initialize(childrenOIs);
}
@Override
public Object evaluate(Object row) throws HiveException {
rowObject = row;
+ if (isEager) {
+ for (int i = 0; i < deferredChildren.length; i++) {
+ ((EagerExprObject) deferredChildren[i]).evaluate();
+ }
+ }
return genericUDF.evaluate(deferredChildren);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Fri Feb 25 03:44:50 2011
@@ -1005,6 +1005,11 @@ public final class FunctionRegistry {
* Returns whether a GenericUDF is deterministic or not.
*/
public static boolean isDeterministic(GenericUDF genericUDF) {
+ if (isStateful(genericUDF)) {
+ // stateful implies non-deterministic, regardless of whatever
+ // the deterministic annotation declares
+ return false;
+ }
UDFType genericUDFType = genericUDF.getClass().getAnnotation(UDFType.class);
if (genericUDFType != null && genericUDFType.deterministic() == false) {
return false;
@@ -1022,6 +1027,26 @@ public final class FunctionRegistry {
}
/**
+ * Returns whether a GenericUDF is stateful or not.
+ */
+ public static boolean isStateful(GenericUDF genericUDF) {
+ UDFType genericUDFType = genericUDF.getClass().getAnnotation(UDFType.class);
+ if (genericUDFType != null && genericUDFType.stateful()) {
+ return true;
+ }
+
+ if (genericUDF instanceof GenericUDFBridge) {
+ GenericUDFBridge bridge = (GenericUDFBridge) genericUDF;
+ UDFType bridgeUDFType = bridge.getUdfClass().getAnnotation(UDFType.class);
+ if (bridgeUDFType != null && bridgeUDFType.stateful()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
* Returns whether the exprNodeDesc is a node of "and", "or", "not".
*/
public static boolean isOpAndOrNot(ExprNodeDesc desc) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Fri Feb 25 03:44:50 2011
@@ -131,6 +131,7 @@ public enum ErrorMsg {
UDTF_LATERAL_VIEW("UDTF's cannot be in a select expression when there is a lateral view"),
UDTF_ALIAS_MISMATCH("The number of aliases supplied in the AS clause does not match the "
+ "number of columns output by the UDTF"),
+ UDF_STATEFUL_INVALID_LOCATION("Stateful UDF's can only be invoked in the SELECT list"),
LATERAL_VIEW_WITH_JOIN("Join with a lateral view is not supported"),
LATERAL_VIEW_INVALID_CHILD("Lateral view AST with invalid child"),
OUTPUT_SPECIFIED_MULTIPLE_TIMES("The same output cannot be present multiple times: "),
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Feb 25 03:44:50 2011
@@ -2050,7 +2050,10 @@ public class SemanticAnalyzer extends Ba
qb.getAliases());
} else {
// Case when this is an expression
- ExprNodeDesc exp = genExprNodeDesc(expr, inputRR);
+ TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR);
+ // We allow stateful functions in the SELECT list (but nowhere else)
+ tcCtx.setAllowStatefulFunctions(true);
+ ExprNodeDesc exp = genExprNodeDesc(expr, inputRR, tcCtx);
col_list.add(exp);
if (!StringUtils.isEmpty(alias)
&& (out_rwsch.get(null, colAlias) != null)) {
@@ -6728,9 +6731,31 @@ public class SemanticAnalyzer extends Ba
* @return exprNodeDesc
* @throws SemanticException
*/
- @SuppressWarnings("nls")
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input)
throws SemanticException {
+ // Since the user didn't supply a customized type-checking context,
+ // use default settings.
+ TypeCheckCtx tcCtx = new TypeCheckCtx(input);
+ return genExprNodeDesc(expr, input, tcCtx);
+ }
+
+ /**
+ * Generates an expression node descriptor for the expression passed in the
+ * arguments. This function uses the row resolver and the metadata information
+ * that are passed as arguments to resolve the column names to internal names.
+ *
+ * @param expr
+ * The expression
+ * @param input
+ * The row resolver
+ * @param tcCtx
+ * Customized type-checking context
+ * @return exprNodeDesc
+ * @throws SemanticException
+ */
+ @SuppressWarnings("nls")
+ public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input,
+ TypeCheckCtx tcCtx) throws SemanticException {
// We recursively create the exprNodeDesc. Base cases: when we encounter
// a column ref, we convert that into an exprNodeColumnDesc; when we
// encounter
@@ -6750,8 +6775,7 @@ public class SemanticAnalyzer extends Ba
.getIsVirtualCol());
}
- // Create the walker, the rules dispatcher and the context.
- TypeCheckCtx tcCtx = new TypeCheckCtx(input);
+ // Create the walker and the rules dispatcher.
tcCtx.setUnparseTranslator(unparseTranslator);
HashMap<Node, Object> nodeOutputs =
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java Fri Feb 25 03:44:50 2011
@@ -48,6 +48,11 @@ public class TypeCheckCtx implements Nod
private ASTNode errorSrcNode;
/**
+ * Whether to allow stateful UDF invocations.
+ */
+ private boolean allowStatefulFunctions;
+
+ /**
* Constructor.
*
* @param inputRR
@@ -56,6 +61,7 @@ public class TypeCheckCtx implements Nod
public TypeCheckCtx(RowResolver inputRR) {
setInputRR(inputRR);
error = null;
+ allowStatefulFunctions = false;
}
/**
@@ -89,6 +95,20 @@ public class TypeCheckCtx implements Nod
}
/**
+ * @param allowStatefulFunctions whether to allow stateful UDF invocations
+ */
+ public void setAllowStatefulFunctions(boolean allowStatefulFunctions) {
+ this.allowStatefulFunctions = allowStatefulFunctions;
+ }
+
+ /**
+ * @return whether to allow stateful UDF invocations
+ */
+ public boolean getAllowStatefulFunctions() {
+ return allowStatefulFunctions;
+ }
+
+ /**
* @param error
* the error to set
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Fri Feb 25 03:44:50 2011
@@ -630,6 +630,12 @@ public final class TypeCheckProcFactory
if (fi.getGenericUDTF() != null) {
throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg());
}
+ if (!ctx.getAllowStatefulFunctions() && (fi.getGenericUDF() != null)) {
+ if (FunctionRegistry.isStateful(fi.getGenericUDF())) {
+ throw new SemanticException(
+ ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg());
+ }
+ }
desc = ExprNodeGenericFuncDesc.newInstance(fi.getGenericUDF(), children);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFType.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFType.java?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFType.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFType.java Fri Feb 25 03:44:50 2011
@@ -32,4 +32,5 @@ import java.lang.annotation.Target;
@Inherited
public @interface UDFType {
boolean deterministic() default true;
+ boolean stateful() default false;
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/udf_case.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udf_case.q?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udf_case.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/udf_case.q Fri Feb 25 03:44:50 2011
@@ -55,3 +55,9 @@ SELECT CASE 1
WHEN 21 THEN 24
END
FROM src LIMIT 1;
+
+-- verify that short-circuiting is working correctly for CASE
+-- we should never get to the ELSE branch, which would raise an exception
+SELECT CASE 1 WHEN 1 THEN 'yo'
+ELSE reflect('java.lang.String', 'bogus', 1) END
+FROM src LIMIT 1;
Modified: hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out?rev=1074393&r1=1074392&r2=1074393&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out Fri Feb 25 03:44:50 2011
@@ -134,7 +134,7 @@ PREHOOK: query: SELECT CASE 1
FROM src LIMIT 1
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-31-04_518_6857103604327967113/-mr-10000
+PREHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-51-38_651_2224865717899515186/-mr-10000
POSTHOOK: query: SELECT CASE 1
WHEN 1 THEN 2
WHEN 3 THEN 4
@@ -163,5 +163,22 @@ POSTHOOK: query: SELECT CASE 1
FROM src LIMIT 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_17-31-04_518_6857103604327967113/-mr-10000
+POSTHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-51-38_651_2224865717899515186/-mr-10000
2 5 15 NULL 20 24
+PREHOOK: query: -- verify that short-circuiting is working correctly for CASE
+-- we should never get to the ELSE branch, which would raise an exception
+SELECT CASE 1 WHEN 1 THEN 'yo'
+ELSE reflect('java.lang.String', 'bogus', 1) END
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-51-44_358_6447239253045741230/-mr-10000
+POSTHOOK: query: -- verify that short-circuiting is working correctly for CASE
+-- we should never get to the ELSE branch, which would raise an exception
+SELECT CASE 1 WHEN 1 THEN 'yo'
+ELSE reflect('java.lang.String', 'bogus', 1) END
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/var/folders/7P/7PeC14kXFIWq0PIYyexGbmKuXUk/-Tmp-/jsichi/hive_2011-02-22_22-51-44_358_6447239253045741230/-mr-10000
+yo