You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2014/09/18 02:56:47 UTC
svn commit: r1625868 - in /hive/branches/cbo/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/
test/queries/clientpositive/ test/results/clientpositive/
Author: sershe
Date: Thu Sep 18 00:56:46 2014
New Revision: 1625868
URL: http://svn.apache.org/r1625868
Log:
HIVE-8080: CBO: function name may not match UDF name during translation (Sergey Shelukhin, reviewed by Laljo John Pullokkaran)
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q
hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Thu Sep 18 00:56:46 2014
@@ -639,6 +639,14 @@ public final class FunctionRegistry {
}
}
+ public static String getNormalizedFunctionName(String fn) {
+ // Does the same thing as getFunctionInfo, except for getting the function info.
+ fn = fn.toLowerCase();
+ return (FunctionUtils.isQualifiedFunctionName(fn) || mFunctions.get(fn) != null) ? fn
+ : FunctionUtils.qualifyFunctionName(
+ fn, SessionState.get().getCurrentDatabase().toLowerCase());
+ }
+
private static <T extends CommonFunctionInfo> T getFunctionInfo(
Map<String, T> mFunctions, String functionName) {
functionName = functionName.toLowerCase();
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java Thu Sep 18 00:56:46 2014
@@ -152,9 +152,7 @@ public class RexNodeConverter {
List<RexNode> childRexNodeLst = new LinkedList<RexNode>();
Builder<RelDataType> argTypeBldr = ImmutableList.<RelDataType> builder();
- // TODO: 1) Expand to other functions as needed 2) What about types
- // other
- // than primitive
+ // TODO: 1) Expand to other functions as needed 2) What about types other than primitive.
if (func.getGenericUDF() instanceof GenericUDFBaseNumeric) {
tgtDT = func.getTypeInfo();
} else if (func.getGenericUDF() instanceof GenericUDFBaseCompare) {
@@ -183,8 +181,8 @@ public class RexNodeConverter {
if (expr == null) {
retType = (expr != null) ? expr.getType() : TypeConverter.convert(func.getTypeInfo(),
m_cluster.getTypeFactory());
- SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getGenericUDF(),
- argTypeBldr.build(), retType);
+ SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(
+ func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType);
expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
} else {
retType = expr.getType();
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Thu Sep 18 00:56:46 2014
@@ -21,6 +21,9 @@ import java.lang.annotation.Annotation;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -61,26 +64,36 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Maps;
public class SqlFunctionConverter {
+ private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class);
+
static final Map<String, SqlOperator> hiveToOptiq;
static final Map<SqlOperator, HiveToken> optiqToHiveToken;
static final Map<SqlOperator, String> reverseOperatorMap;
static {
- Builder builder = new Builder();
+ StaticBlockBuilder builder = new StaticBlockBuilder();
hiveToOptiq = ImmutableMap.copyOf(builder.hiveToOptiq);
optiqToHiveToken = ImmutableMap.copyOf(builder.optiqToHiveToken);
reverseOperatorMap = ImmutableMap.copyOf(builder.reverseOperatorMap);
}
- public static SqlOperator getOptiqOperator(GenericUDF hiveUDF,
+ public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveUDF,
ImmutableList<RelDataType> optiqArgTypes, RelDataType retType) throws OptiqSemanticException {
// handle overloaded methods first
if (hiveUDF instanceof GenericUDFOPNegative) {
return SqlStdOperatorTable.UNARY_MINUS;
} else if (hiveUDF instanceof GenericUDFOPPositive) {
return SqlStdOperatorTable.UNARY_PLUS;
- } // do genric lookup
- return getOptiqFn(getName(hiveUDF), optiqArgTypes, retType);
+ } // do generic lookup
+ String name = null;
+ if (StringUtils.isEmpty(funcTextName)) {
+ name = getName(hiveUDF); // this should probably never happen, see getName comment
+ LOG.warn("The function text was empty, name from annotation is " + name);
+ } else {
+ // We could just do toLowerCase here and let SA qualify it, but let's be proper...
+ name = FunctionRegistry.getNormalizedFunctionName(funcTextName);
+ }
+ return getOptiqFn(name, optiqArgTypes, retType);
}
public static GenericUDF getHiveUDF(SqlOperator op, RelDataType dt) {
@@ -197,6 +210,9 @@ public class SqlFunctionConverter {
}
+ // TODO: this is not valid. Function names for built-in UDFs are specified in FunctionRegistry,
+ // and only happen to match annotations. For user UDFs, the name is what user specifies at
+ // creation time (annotation can be absent, different, or duplicate some other function).
private static String getName(GenericUDF hiveUDF) {
String udfName = null;
if (hiveUDF instanceof GenericUDFBridge) {
@@ -228,12 +244,13 @@ public class SqlFunctionConverter {
return udfName;
}
- private static class Builder {
+ /** This class is used to build immutable hashmaps in the static block above. */
+ private static class StaticBlockBuilder {
final Map<String, SqlOperator> hiveToOptiq = Maps.newHashMap();
final Map<SqlOperator, HiveToken> optiqToHiveToken = Maps.newHashMap();
final Map<SqlOperator, String> reverseOperatorMap = Maps.newHashMap();
- Builder() {
+ StaticBlockBuilder() {
registerFunction("+", SqlStdOperatorTable.PLUS, hToken(HiveParser.PLUS, "+"));
registerFunction("-", SqlStdOperatorTable.MINUS, hToken(HiveParser.MINUS, "-"));
registerFunction("*", SqlStdOperatorTable.MULTIPLY, hToken(HiveParser.STAR, "*"));
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q Thu Sep 18 00:56:46 2014
@@ -2,11 +2,16 @@
-- qtest_get_java_boolean should already be created during test initialization
select qtest_get_java_boolean('true'), qtest_get_java_boolean('false') from src limit 1;
+describe function extended qtest_get_java_boolean;
+
create database mydb;
create function mydb.func1 as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper';
show functions mydb.func1;
+describe function extended mydb.func1;
+
+
select mydb.func1('abc') from src limit 1;
drop function mydb.func1;
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out Thu Sep 18 00:56:46 2014
@@ -9,6 +9,12 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
true false
+PREHOOK: query: describe function extended qtest_get_java_boolean
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function extended qtest_get_java_boolean
+POSTHOOK: type: DESCFUNCTION
+qtest_get_java_boolean(str) - GenericUDF to return native Java's boolean type
+Synonyms: default.qtest_get_java_boolean
PREHOOK: query: create database mydb
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:mydb
@@ -28,6 +34,15 @@ PREHOOK: type: SHOWFUNCTIONS
POSTHOOK: query: show functions mydb.func1
POSTHOOK: type: SHOWFUNCTIONS
mydb.func1
+PREHOOK: query: describe function extended mydb.func1
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function extended mydb.func1
+POSTHOOK: type: DESCFUNCTION
+mydb.func1(str) - Returns str with all characters changed to uppercase
+Synonyms: upper, ucase
+Example:
+ > SELECT mydb.func1('Facebook') FROM src LIMIT 1;
+ 'FACEBOOK'
PREHOOK: query: select mydb.func1('abc') from src limit 1
PREHOOK: type: QUERY
PREHOOK: Input: default@src