You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/09/13 01:21:04 UTC
svn commit: r1624678 - in /hive/branches/cbo/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/
test/queries/clientpositive/ test/results/clientpositive/
Author: hashutosh
Date: Fri Sep 12 23:21:03 2014
New Revision: 1624678
URL: http://svn.apache.org/r1624678
Log:
HIVE-8080 : CBO: function name may not match UDF name during translation (Sergey Shelukhin via John Pullokkaran)
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q
hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1624678&r1=1624677&r2=1624678&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java Fri Sep 12 23:21:03 2014
@@ -152,9 +152,7 @@ public class RexNodeConverter {
List<RexNode> childRexNodeLst = new LinkedList<RexNode>();
Builder<RelDataType> argTypeBldr = ImmutableList.<RelDataType> builder();
- // TODO: 1) Expand to other functions as needed 2) What about types
- // other
- // than primitive
+ // TODO: 1) Expand to other functions as needed 2) What about types other than primitive.
if (func.getGenericUDF() instanceof GenericUDFBaseNumeric) {
tgtDT = func.getTypeInfo();
} else if (func.getGenericUDF() instanceof GenericUDFBaseCompare) {
@@ -183,8 +181,8 @@ public class RexNodeConverter {
if (expr == null) {
retType = (expr != null) ? expr.getType() : TypeConverter.convert(func.getTypeInfo(),
m_cluster.getTypeFactory());
- SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getGenericUDF(),
- argTypeBldr.build(), retType);
+ SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(
+ func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType);
expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
} else {
retType = expr.getType();
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1624678&r1=1624677&r2=1624678&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Fri Sep 12 23:21:03 2014
@@ -21,6 +21,7 @@ import java.lang.annotation.Annotation;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -66,21 +67,22 @@ public class SqlFunctionConverter {
static final Map<SqlOperator, String> reverseOperatorMap;
static {
- Builder builder = new Builder();
+ StaticBlockBuilder builder = new StaticBlockBuilder();
hiveToOptiq = ImmutableMap.copyOf(builder.hiveToOptiq);
optiqToHiveToken = ImmutableMap.copyOf(builder.optiqToHiveToken);
reverseOperatorMap = ImmutableMap.copyOf(builder.reverseOperatorMap);
}
- public static SqlOperator getOptiqOperator(GenericUDF hiveUDF,
+ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveUDF,
ImmutableList<RelDataType> optiqArgTypes, RelDataType retType) throws OptiqSemanticException {
// handle overloaded methods first
if (hiveUDF instanceof GenericUDFOPNegative) {
return SqlStdOperatorTable.UNARY_MINUS;
} else if (hiveUDF instanceof GenericUDFOPPositive) {
return SqlStdOperatorTable.UNARY_PLUS;
- } // do genric lookup
- return getOptiqFn(getName(hiveUDF), optiqArgTypes, retType);
+ } // do generic lookup
+ String name = StringUtils.isEmpty(funcTextName) ? getName(hiveUDF) : funcTextName;
+ return getOptiqFn(name, optiqArgTypes, retType);
}
public static GenericUDF getHiveUDF(SqlOperator op, RelDataType dt) {
@@ -197,6 +199,9 @@ public class SqlFunctionConverter {
}
+ // TODO: this is not valid. Function names for built-in UDFs are specified in FunctionRegistry,
+ // and only happen to match annotations. For user UDFs, the name is what user specifies at
+ // creation time (annotation can be absent, different, or duplicate some other function).
private static String getName(GenericUDF hiveUDF) {
String udfName = null;
if (hiveUDF instanceof GenericUDFBridge) {
@@ -228,12 +233,13 @@ public class SqlFunctionConverter {
return udfName;
}
- private static class Builder {
+ /** This class is used to build immutable hashmaps in the static block above. */
+ private static class StaticBlockBuilder {
final Map<String, SqlOperator> hiveToOptiq = Maps.newHashMap();
final Map<SqlOperator, HiveToken> optiqToHiveToken = Maps.newHashMap();
final Map<SqlOperator, String> reverseOperatorMap = Maps.newHashMap();
- Builder() {
+ StaticBlockBuilder() {
registerFunction("+", SqlStdOperatorTable.PLUS, hToken(HiveParser.PLUS, "+"));
registerFunction("-", SqlStdOperatorTable.MINUS, hToken(HiveParser.MINUS, "-"));
registerFunction("*", SqlStdOperatorTable.MULTIPLY, hToken(HiveParser.STAR, "*"));
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q?rev=1624678&r1=1624677&r2=1624678&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q Fri Sep 12 23:21:03 2014
@@ -2,11 +2,16 @@
-- qtest_get_java_boolean should already be created during test initialization
select qtest_get_java_boolean('true'), qtest_get_java_boolean('false') from src limit 1;
+describe function extended qtest_get_java_boolean;
+
create database mydb;
create function mydb.func1 as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper';
show functions mydb.func1;
+describe function extended mydb.func1;
+
+
select mydb.func1('abc') from src limit 1;
drop function mydb.func1;
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out?rev=1624678&r1=1624677&r2=1624678&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out Fri Sep 12 23:21:03 2014
@@ -9,6 +9,12 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
true false
+PREHOOK: query: describe function extended qtest_get_java_boolean
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function extended qtest_get_java_boolean
+POSTHOOK: type: DESCFUNCTION
+qtest_get_java_boolean(str) - GenericUDF to return native Java's boolean type
+Synonyms: default.qtest_get_java_boolean
PREHOOK: query: create database mydb
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:mydb
@@ -28,6 +34,15 @@ PREHOOK: type: SHOWFUNCTIONS
POSTHOOK: query: show functions mydb.func1
POSTHOOK: type: SHOWFUNCTIONS
mydb.func1
+PREHOOK: query: describe function extended mydb.func1
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function extended mydb.func1
+POSTHOOK: type: DESCFUNCTION
+mydb.func1(str) - Returns str with all characters changed to uppercase
+Synonyms: upper, ucase
+Example:
+ > SELECT mydb.func1('Facebook') FROM src LIMIT 1;
+ 'FACEBOOK'
PREHOOK: query: select mydb.func1('abc') from src limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@src