You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/08/14 09:48:09 UTC
svn commit: r804106 [2/24] - in /hadoop/hive/trunk: ./
contrib/src/java/org/apache/hadoop/hive/contrib/udaf/
contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/
contrib/src/java/org/apache/hadoop/hive/contrib/udf/
contrib/src/java/org/apache/...
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Fri Aug 14 07:48:02 2009
@@ -33,14 +33,17 @@
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
-import org.apache.hadoop.hive.ql.exec.FunctionInfo.OperatorType;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.groupByDesc;
import org.apache.hadoop.hive.ql.udf.*;
import org.apache.hadoop.hive.ql.udf.generic.*;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -56,102 +59,102 @@
static LinkedHashMap<String, FunctionInfo> mFunctions;
static {
mFunctions = new LinkedHashMap<String, FunctionInfo>();
- registerUDF("concat", UDFConcat.class, OperatorType.PREFIX, false);
- registerUDF("substr", UDFSubstr.class, OperatorType.PREFIX, false);
- registerUDF("substring", UDFSubstr.class, OperatorType.PREFIX, false);
- registerUDF("space", UDFSpace.class, OperatorType.PREFIX, false);
- registerUDF("repeat", UDFRepeat.class, OperatorType.PREFIX, false);
- registerUDF("ascii", UDFAscii.class, OperatorType.PREFIX, false);
- registerUDF("lpad", UDFLpad.class, OperatorType.PREFIX, false);
- registerUDF("rpad", UDFRpad.class, OperatorType.PREFIX, false);
+ registerUDF("concat", UDFConcat.class, false);
+ registerUDF("substr", UDFSubstr.class, false);
+ registerUDF("substring", UDFSubstr.class, false);
+ registerUDF("space", UDFSpace.class, false);
+ registerUDF("repeat", UDFRepeat.class, false);
+ registerUDF("ascii", UDFAscii.class, false);
+ registerUDF("lpad", UDFLpad.class, false);
+ registerUDF("rpad", UDFRpad.class, false);
registerGenericUDF("size", GenericUDFSize.class);
- registerUDF("round", UDFRound.class, OperatorType.PREFIX, false);
- registerUDF("floor", UDFFloor.class, OperatorType.PREFIX, false);
- registerUDF("sqrt", UDFSqrt.class, OperatorType.PREFIX, false);
- registerUDF("ceil", UDFCeil.class, OperatorType.PREFIX, false);
- registerUDF("ceiling", UDFCeil.class, OperatorType.PREFIX, false);
- registerUDF("rand", UDFRand.class, OperatorType.PREFIX, false);
- registerUDF("abs", UDFAbs.class, OperatorType.PREFIX, false);
- registerUDF("pmod", UDFPosMod.class, OperatorType.PREFIX, false);
-
- registerUDF("ln", UDFLn.class, OperatorType.PREFIX, false);
- registerUDF("log2", UDFLog2.class, OperatorType.PREFIX, false);
- registerUDF("sin",UDFSin.class, OperatorType.PREFIX, false);
- registerUDF("asin",UDFAsin.class, OperatorType.PREFIX, false);
- registerUDF("cos",UDFCos.class, OperatorType.PREFIX, false);
- registerUDF("acos",UDFAcos.class, OperatorType.PREFIX, false);
- registerUDF("log10", UDFLog10.class, OperatorType.PREFIX, false);
- registerUDF("log", UDFLog.class, OperatorType.PREFIX, false);
- registerUDF("exp", UDFExp.class, OperatorType.PREFIX, false);
- registerUDF("power", UDFPower.class, OperatorType.PREFIX, false);
- registerUDF("pow", UDFPower.class, OperatorType.PREFIX, false);
-
- registerUDF("conv", UDFConv.class, OperatorType.PREFIX, false);
- registerUDF("bin", UDFBin.class, OperatorType.PREFIX, false);
- registerUDF("hex", UDFHex.class, OperatorType.PREFIX, false);
+ registerUDF("round", UDFRound.class, false);
+ registerUDF("floor", UDFFloor.class, false);
+ registerUDF("sqrt", UDFSqrt.class, false);
+ registerUDF("ceil", UDFCeil.class, false);
+ registerUDF("ceiling", UDFCeil.class, false);
+ registerUDF("rand", UDFRand.class, false);
+ registerUDF("abs", UDFAbs.class, false);
+ registerUDF("pmod", UDFPosMod.class, false);
+
+ registerUDF("ln", UDFLn.class, false);
+ registerUDF("log2", UDFLog2.class, false);
+ registerUDF("sin",UDFSin.class, false);
+ registerUDF("asin",UDFAsin.class, false);
+ registerUDF("cos",UDFCos.class, false);
+ registerUDF("acos",UDFAcos.class, false);
+ registerUDF("log10", UDFLog10.class, false);
+ registerUDF("log", UDFLog.class, false);
+ registerUDF("exp", UDFExp.class, false);
+ registerUDF("power", UDFPower.class, false);
+ registerUDF("pow", UDFPower.class, false);
+
+ registerUDF("conv", UDFConv.class, false);
+ registerUDF("bin", UDFBin.class, false);
+ registerUDF("hex", UDFHex.class, false);
- registerUDF("upper", UDFUpper.class, OperatorType.PREFIX, false);
- registerUDF("lower", UDFLower.class, OperatorType.PREFIX, false);
- registerUDF("ucase", UDFUpper.class, OperatorType.PREFIX, false);
- registerUDF("lcase", UDFLower.class, OperatorType.PREFIX, false);
- registerUDF("trim", UDFTrim.class, OperatorType.PREFIX, false);
- registerUDF("ltrim", UDFLTrim.class, OperatorType.PREFIX, false);
- registerUDF("rtrim", UDFRTrim.class, OperatorType.PREFIX, false);
- registerUDF("length", UDFLength.class, OperatorType.PREFIX, false);
- registerUDF("reverse", UDFReverse.class, OperatorType.PREFIX, false);
-
- registerUDF("like", UDFLike.class, OperatorType.INFIX, true);
- registerUDF("rlike", UDFRegExp.class, OperatorType.INFIX, true);
- registerUDF("regexp", UDFRegExp.class, OperatorType.INFIX, true);
- registerUDF("regexp_replace", UDFRegExpReplace.class, OperatorType.PREFIX, false);
- registerUDF("regexp_extract", UDFRegExpExtract.class, OperatorType.PREFIX, false);
- registerUDF("parse_url", UDFParseUrl.class, OperatorType.PREFIX, false);
+ registerUDF("upper", UDFUpper.class, false);
+ registerUDF("lower", UDFLower.class, false);
+ registerUDF("ucase", UDFUpper.class, false);
+ registerUDF("lcase", UDFLower.class, false);
+ registerUDF("trim", UDFTrim.class, false);
+ registerUDF("ltrim", UDFLTrim.class, false);
+ registerUDF("rtrim", UDFRTrim.class, false);
+ registerUDF("length", UDFLength.class, false);
+ registerUDF("reverse", UDFReverse.class, false);
+
+ registerUDF("like", UDFLike.class, true);
+ registerUDF("rlike", UDFRegExp.class, true);
+ registerUDF("regexp", UDFRegExp.class, true);
+ registerUDF("regexp_replace", UDFRegExpReplace.class, false);
+ registerUDF("regexp_extract", UDFRegExpExtract.class, false);
+ registerUDF("parse_url", UDFParseUrl.class, false);
registerGenericUDF("split", GenericUDFSplit.class);
- registerUDF("positive", UDFOPPositive.class, OperatorType.PREFIX, true, "+");
- registerUDF("negative", UDFOPNegative.class, OperatorType.PREFIX, true, "-");
+ registerUDF("positive", UDFOPPositive.class, true, "+");
+ registerUDF("negative", UDFOPNegative.class, true, "-");
- registerUDF("day", UDFDayOfMonth.class, OperatorType.PREFIX, false);
- registerUDF("dayofmonth", UDFDayOfMonth.class, OperatorType.PREFIX, false);
- registerUDF("month", UDFMonth.class, OperatorType.PREFIX, false);
- registerUDF("year", UDFYear.class, OperatorType.PREFIX, false);
- registerUDF("from_unixtime", UDFFromUnixTime.class, OperatorType.PREFIX, false);
- registerUDF("unix_timestamp", UDFUnixTimeStamp.class, OperatorType.PREFIX, false);
- registerUDF("to_date", UDFDate.class, OperatorType.PREFIX, false);
-
- registerUDF("date_add", UDFDateAdd.class, OperatorType.PREFIX, false);
- registerUDF("date_sub", UDFDateSub.class, OperatorType.PREFIX, false);
- registerUDF("datediff", UDFDateDiff.class, OperatorType.PREFIX, false);
-
- registerUDF("get_json_object", UDFJson.class, OperatorType.PREFIX, false);
-
- registerUDF("+", UDFOPPlus.class, OperatorType.INFIX, true);
- registerUDF("-", UDFOPMinus.class, OperatorType.INFIX, true);
- registerUDF("*", UDFOPMultiply.class, OperatorType.INFIX, true);
- registerUDF("/", UDFOPDivide.class, OperatorType.INFIX, true);
- registerUDF("%", UDFOPMod.class, OperatorType.INFIX, true);
-
- registerUDF("&", UDFOPBitAnd.class, OperatorType.INFIX, true);
- registerUDF("|", UDFOPBitOr.class, OperatorType.INFIX, true);
- registerUDF("^", UDFOPBitXor.class, OperatorType.INFIX, true);
- registerUDF("~", UDFOPBitNot.class, OperatorType.PREFIX, true);
-
- registerUDF("=", UDFOPEqual.class, OperatorType.INFIX, true);
- registerUDF("==", UDFOPEqual.class, OperatorType.INFIX, true, "=");
- registerUDF("<>", UDFOPNotEqual.class, OperatorType.INFIX, true);
- registerUDF("<", UDFOPLessThan.class, OperatorType.INFIX, true);
- registerUDF("<=", UDFOPEqualOrLessThan.class, OperatorType.INFIX, true);
- registerUDF(">", UDFOPGreaterThan.class, OperatorType.INFIX, true);
- registerUDF(">=", UDFOPEqualOrGreaterThan.class, OperatorType.INFIX, true);
-
- registerUDF("and", UDFOPAnd.class, OperatorType.INFIX, true);
- registerUDF("&&", UDFOPAnd.class, OperatorType.INFIX, true, "and");
- registerUDF("or", UDFOPOr.class, OperatorType.INFIX, true);
- registerUDF("||", UDFOPOr.class, OperatorType.INFIX, true, "or");
- registerUDF("not", UDFOPNot.class, OperatorType.PREFIX, true);
- registerUDF("!", UDFOPNot.class, OperatorType.PREFIX, true, "not");
+ registerUDF("day", UDFDayOfMonth.class, false);
+ registerUDF("dayofmonth", UDFDayOfMonth.class, false);
+ registerUDF("month", UDFMonth.class, false);
+ registerUDF("year", UDFYear.class, false);
+ registerUDF("from_unixtime", UDFFromUnixTime.class, false);
+ registerUDF("unix_timestamp", UDFUnixTimeStamp.class, false);
+ registerUDF("to_date", UDFDate.class, false);
+
+ registerUDF("date_add", UDFDateAdd.class, false);
+ registerUDF("date_sub", UDFDateSub.class, false);
+ registerUDF("datediff", UDFDateDiff.class, false);
+
+ registerUDF("get_json_object", UDFJson.class, false);
+
+ registerUDF("+", UDFOPPlus.class, true);
+ registerUDF("-", UDFOPMinus.class, true);
+ registerUDF("*", UDFOPMultiply.class, true);
+ registerUDF("/", UDFOPDivide.class, true);
+ registerUDF("%", UDFOPMod.class, true);
+
+ registerUDF("&", UDFOPBitAnd.class, true);
+ registerUDF("|", UDFOPBitOr.class, true);
+ registerUDF("^", UDFOPBitXor.class, true);
+ registerUDF("~", UDFOPBitNot.class, true);
+
+ registerUDF("=", UDFOPEqual.class, true);
+ registerUDF("==", UDFOPEqual.class, true, "=");
+ registerUDF("<>", UDFOPNotEqual.class, true);
+ registerUDF("<", UDFOPLessThan.class, true);
+ registerUDF("<=", UDFOPEqualOrLessThan.class, true);
+ registerUDF(">", UDFOPGreaterThan.class, true);
+ registerUDF(">=", UDFOPEqualOrGreaterThan.class, true);
+
+ registerUDF("and", UDFOPAnd.class, true);
+ registerUDF("&&", UDFOPAnd.class, true, "and");
+ registerUDF("or", UDFOPOr.class, true);
+ registerUDF("||", UDFOPOr.class, true, "or");
+ registerUDF("not", UDFOPNot.class, true);
+ registerUDF("!", UDFOPNot.class, true, "not");
registerGenericUDF("isnull", GenericUDFOPNull.class);
registerGenericUDF("isnotnull", GenericUDFOPNotNull.class);
@@ -160,21 +163,21 @@
// Aliases for Java Class Names
// These are used in getImplicitConvertUDFMethod
- registerUDF(Constants.BOOLEAN_TYPE_NAME, UDFToBoolean.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.BOOLEAN_TYPE_NAME, UDFToBoolean.class, false,
UDFToBoolean.class.getSimpleName());
- registerUDF(Constants.TINYINT_TYPE_NAME, UDFToByte.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.TINYINT_TYPE_NAME, UDFToByte.class, false,
UDFToByte.class.getSimpleName());
- registerUDF(Constants.SMALLINT_TYPE_NAME, UDFToShort.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.SMALLINT_TYPE_NAME, UDFToShort.class, false,
UDFToShort.class.getSimpleName());
- registerUDF(Constants.INT_TYPE_NAME, UDFToInteger.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.INT_TYPE_NAME, UDFToInteger.class, false,
UDFToInteger.class.getSimpleName());
- registerUDF(Constants.BIGINT_TYPE_NAME, UDFToLong.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.BIGINT_TYPE_NAME, UDFToLong.class, false,
UDFToLong.class.getSimpleName());
- registerUDF(Constants.FLOAT_TYPE_NAME, UDFToFloat.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.FLOAT_TYPE_NAME, UDFToFloat.class, false,
UDFToFloat.class.getSimpleName());
- registerUDF(Constants.DOUBLE_TYPE_NAME, UDFToDouble.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.DOUBLE_TYPE_NAME, UDFToDouble.class, false,
UDFToDouble.class.getSimpleName());
- registerUDF(Constants.STRING_TYPE_NAME, UDFToString.class, OperatorType.PREFIX, false,
+ registerUDF(Constants.STRING_TYPE_NAME, UDFToString.class, false,
UDFToString.class.getSimpleName());
// Aggregate functions
@@ -204,46 +207,31 @@
registerGenericUDF("elt", GenericUDFElt.class);
}
- public static FunctionInfo getUDFInfo(Class<?> fClass) {
- for(Map.Entry<String, FunctionInfo> ent: mFunctions.entrySet()) {
- FunctionInfo val = ent.getValue();
- if (val.getUDFClass() == fClass) {
- return val;
- }
- }
-
- return null;
- }
-
public static void registerTemporaryUDF(String functionName, Class<? extends UDF> UDFClass,
- FunctionInfo.OperatorType opt, boolean isOperator) {
- registerUDF(false,functionName, UDFClass, opt, isOperator);
+ boolean isOperator) {
+ registerUDF(false, functionName, UDFClass, isOperator);
}
static void registerUDF(String functionName, Class<? extends UDF> UDFClass,
- FunctionInfo.OperatorType opt, boolean isOperator) {
- registerUDF(true,functionName, UDFClass, opt, isOperator);
+ boolean isOperator) {
+ registerUDF(true, functionName, UDFClass, isOperator);
}
public static void registerUDF(boolean isNative, String functionName, Class<? extends UDF> UDFClass,
- FunctionInfo.OperatorType opt, boolean isOperator) {
- if (UDF.class.isAssignableFrom(UDFClass)) {
- FunctionInfo fI = new FunctionInfo(isNative, functionName.toLowerCase(), UDFClass, null);
- fI.setIsOperator(isOperator);
- fI.setOpType(opt);
- mFunctions.put(functionName.toLowerCase(), fI);
- } else {
- throw new RuntimeException("Registering UDF Class " + UDFClass + " which does not extends " + UDF.class);
- }
+ boolean isOperator) {
+ registerUDF(isNative, functionName, UDFClass, isOperator, functionName.toLowerCase());
}
public static void registerUDF(String functionName, Class<? extends UDF> UDFClass,
- FunctionInfo.OperatorType opt, boolean isOperator,
- String displayName) {
+ boolean isOperator, String displayName) {
+ registerUDF(true, functionName, UDFClass, isOperator, displayName);
+ }
+
+ public static void registerUDF(boolean isNative, String functionName, Class<? extends UDF> UDFClass,
+ boolean isOperator, String displayName) {
if (UDF.class.isAssignableFrom(UDFClass)) {
- FunctionInfo fI = new FunctionInfo(displayName, UDFClass, null);
- fI.setIsOperator(isOperator);
- fI.setOpType(opt);
+ FunctionInfo fI = new FunctionInfo(isNative, displayName,
+ new GenericUDFBridge(displayName, isOperator, UDFClass));
mFunctions.put(functionName.toLowerCase(), fI);
} else {
throw new RuntimeException("Registering UDF Class " + UDFClass + " which does not extends " + UDF.class);
@@ -260,7 +248,8 @@
public static void registerGenericUDF(boolean isNative, String functionName, Class<? extends GenericUDF> genericUDFClass) {
if (GenericUDF.class.isAssignableFrom(genericUDFClass)) {
- FunctionInfo fI = new FunctionInfo(isNative, functionName, null, genericUDFClass);
+ FunctionInfo fI = new FunctionInfo(isNative, functionName,
+ (GenericUDF)ReflectionUtils.newInstance(genericUDFClass, null));
mFunctions.put(functionName.toLowerCase(), fI);
} else {
throw new RuntimeException("Registering GenericUDF Class " + genericUDFClass
@@ -304,16 +293,6 @@
return funcNames;
}
- public static Class<? extends UDF> getUDFClass(String functionName) {
- LOG.debug("Looking up: " + functionName);
- FunctionInfo finfo = mFunctions.get(functionName.toLowerCase());
- if (finfo == null) {
- return null;
- }
- Class<? extends UDF> result = finfo.getUDFClass();
- return result;
- }
-
static Map<TypeInfo, Integer> numericTypes = new HashMap<TypeInfo, Integer>();
static List<TypeInfo> numericTypeList = new ArrayList<TypeInfo>();
static void registerNumericType(String typeName, int level) {
@@ -396,27 +375,6 @@
}
/**
- * Get the UDF method for the name and argumentClasses.
- * @param name the name of the UDF
- * @param argumentTypeInfos
- * @return The UDF method
- */
- public static Method getUDFMethod(String name, List<TypeInfo> argumentTypeInfos) {
- Class<? extends UDF> udf = getUDFClass(name);
- if (udf == null) return null;
- Method udfMethod = null;
- try {
- udfMethod = udf.newInstance().getResolver().getEvalMethod(argumentTypeInfos);
- }
- catch (AmbiguousMethodException e) {
- }
- catch (Exception e) {
- throw new RuntimeException("Cannot get UDF for " + name + " " + argumentTypeInfos, e);
- }
- return udfMethod;
- }
-
- /**
* Get the GenericUDAF evaluator for the name and argumentClasses.
* @param name the name of the UDAF
* @param argumentTypeInfos
@@ -438,7 +396,8 @@
* This method is shared between UDFRegistry and UDAFRegistry.
* methodName will be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial" for UDAFRegistry.
*/
- public static <T> Method getMethodInternal(Class<? extends T> udfClass, String methodName, boolean exact, List<TypeInfo> argumentClasses) {
+ public static <T> Method getMethodInternal(Class<? extends T> udfClass, String methodName, boolean exact,
+ List<TypeInfo> argumentClasses) {
ArrayList<Method> mlist = new ArrayList<Method>();
@@ -451,10 +410,6 @@
return getMethodInternal(mlist, exact, argumentClasses);
}
- public static Method getUDFMethod(String name, TypeInfo ... argumentClasses) {
- return getUDFMethod(name, Arrays.asList(argumentClasses));
- }
-
public static void registerTemporaryGenericUDAF(String functionName, GenericUDAFResolver genericUDAFResolver) {
registerGenericUDAF(false, functionName, genericUDAFResolver);
}
@@ -539,6 +494,51 @@
}
/**
+ * Returns -1 if passed does not match accepted.
+ * Otherwise return the cost (usually 0 for no conversion and 1 for conversion).
+ */
+ public static int matchCost(TypeInfo argumentPassed, TypeInfo argumentAccepted, boolean exact) {
+ if (argumentAccepted.equals(argumentPassed)) {
+ // matches
+ return 0;
+ }
+ if (argumentPassed.equals(TypeInfoFactory.voidTypeInfo)) {
+ // passing null matches everything
+ return 0;
+ }
+ if (argumentPassed.getCategory().equals(Category.LIST)
+ && argumentAccepted.getCategory().equals(Category.LIST)) {
+ // lists are compatible if and only-if the elements are compatible
+ TypeInfo argumentPassedElement = ((ListTypeInfo)argumentPassed).getListElementTypeInfo();
+ TypeInfo argumentAcceptedElement = ((ListTypeInfo)argumentAccepted).getListElementTypeInfo();
+ return matchCost(argumentPassedElement, argumentAcceptedElement, exact);
+ }
+ if (argumentPassed.getCategory().equals(Category.MAP)
+ && argumentAccepted.getCategory().equals(Category.MAP)) {
+ // lists are compatible if and only-if the elements are compatible
+ TypeInfo argumentPassedKey = ((MapTypeInfo)argumentPassed).getMapKeyTypeInfo();
+ TypeInfo argumentAcceptedKey = ((MapTypeInfo)argumentAccepted).getMapKeyTypeInfo();
+ TypeInfo argumentPassedValue = ((MapTypeInfo)argumentPassed).getMapValueTypeInfo();
+ TypeInfo argumentAcceptedValue = ((MapTypeInfo)argumentAccepted).getMapValueTypeInfo();
+ int cost1 = matchCost(argumentPassedKey, argumentAcceptedKey, exact);
+ int cost2 = matchCost(argumentPassedValue, argumentAcceptedValue, exact);
+ if (cost1 < 0 || cost2 < 0) return -1;
+ return Math.max(cost1, cost2);
+ }
+
+ if (argumentAccepted.equals(TypeInfoFactory.unknownTypeInfo)) {
+ // accepting Object means accepting everything,
+ // but there is a conversion cost.
+ return 1;
+ }
+ if (!exact && implicitConvertable(argumentPassed, argumentAccepted)) {
+ return 1;
+ }
+
+ return -1;
+ }
+
+ /**
* Gets the closest matching method corresponding to the argument list from a list of methods.
*
* @param mlist The list of methods to inspect.
@@ -548,56 +548,43 @@
*/
public static Method getMethodInternal(ArrayList<Method> mlist, boolean exact,
List<TypeInfo> argumentsPassed) {
- int leastImplicitConversions = Integer.MAX_VALUE;
+ int leastConversionCost = Integer.MAX_VALUE;
Method udfMethod = null;
- for(Method m: mlist) {
- List<TypeInfo> argumentsAccepted = TypeInfoUtils.getParameterTypeInfos(m);
+ for (Method m: mlist) {
+ List<TypeInfo> argumentsAccepted = TypeInfoUtils.getParameterTypeInfos(m,
+ argumentsPassed.size());
+ if (argumentsAccepted == null) {
+ // null means the method does not accept number of arguments passed.
+ continue;
+ }
boolean match = (argumentsAccepted.size() == argumentsPassed.size());
- int implicitConversions = 0;
+ int conversionCost = 0;
for(int i=0; i<argumentsPassed.size() && match; i++) {
- TypeInfo argumentPassed = argumentsPassed.get(i);
- TypeInfo argumentAccepted = argumentsAccepted.get(i);
- if (argumentPassed.equals(TypeInfoFactory.voidTypeInfo)) {
- // passing null matches everything
- continue;
- }
- if (argumentAccepted.equals(TypeInfoFactory.unknownTypeInfo)) {
- // accepting Object means accepting everything
- continue;
- }
- if (argumentPassed.getCategory().equals(Category.LIST)
- && argumentAccepted.equals(TypeInfoFactory.unknownListTypeInfo)) {
- // accepting List means accepting List of everything
- continue;
- }
- if (argumentPassed.getCategory().equals(Category.MAP)
- && argumentAccepted.equals(TypeInfoFactory.unknownMapTypeInfo)) {
- // accepting Map means accepting Map of everything
- continue;
- }
- TypeInfo accepted = argumentsAccepted.get(i);
- if (accepted.equals(argumentsPassed.get(i))) {
- // do nothing if match
- } else if (!exact && implicitConvertable(argumentsPassed.get(i), accepted)) {
- implicitConversions ++;
- } else {
+ int cost = matchCost(argumentsPassed.get(i), argumentsAccepted.get(i), exact);
+ if (cost == -1) {
match = false;
+ } else {
+ conversionCost += cost;
}
}
-
+ LOG.info("Method " + (match ? "did": "didn't") + " match: passed = " + argumentsPassed
+ + " accepted = " + argumentsAccepted + " method = " + m);
if (match) {
// Always choose the function with least implicit conversions.
- if (implicitConversions < leastImplicitConversions) {
+ if (conversionCost < leastConversionCost) {
udfMethod = m;
- leastImplicitConversions = implicitConversions;
+ leastConversionCost = conversionCost;
// Found an exact match
- if (leastImplicitConversions == 0) break;
- } else if (implicitConversions == leastImplicitConversions){
+ if (leastConversionCost == 0) break;
+ } else if (conversionCost == leastConversionCost){
// Ambiguous call: two methods with the same number of implicit conversions
+ LOG.info("Ambigious methods: passed = " + argumentsPassed
+ + " method 1 = " + udfMethod + " method 2 = " + m);
udfMethod = null;
+ // Don't break! We might find a better match later.
} else {
// do nothing if implicitConversions > leastImplicitConversions
}
@@ -607,10 +594,92 @@
}
/**
- * A shortcut to get the index GenericUDFClass.
+ * A shortcut to get the "index" GenericUDF.
* This is used for getting elements out of array and getting values out of map.
*/
- public static Class<? extends GenericUDF> getGenericUDFClassForIndex() {
- return FunctionRegistry.getFunctionInfo("index").getGenericUDFClass();
+ public static GenericUDF getGenericUDFForIndex() {
+ return FunctionRegistry.getFunctionInfo("index").getGenericUDF();
+ }
+
+ /**
+ * A shortcut to get the "and" GenericUDF.
+ */
+ public static GenericUDF getGenericUDFForAnd() {
+ return FunctionRegistry.getFunctionInfo("and").getGenericUDF();
+ }
+
+ /**
+ * Create a copy of an existing GenericUDF.
+ */
+ public static GenericUDF cloneGenericUDF(GenericUDF genericUDF) {
+ if (genericUDF instanceof GenericUDFBridge) {
+ GenericUDFBridge bridge = (GenericUDFBridge)genericUDF;
+ return new GenericUDFBridge(bridge.getUdfName(), bridge.isOperator(), bridge.getUdfClass());
+ } else {
+ return (GenericUDF)ReflectionUtils.newInstance(genericUDF.getClass(), null);
+ }
+ }
+
+ /**
+ * Get the UDF class from an exprNodeDesc.
+ * Returns null if the exprNodeDesc does not contain a UDF class.
+ */
+ private static Class<? extends UDF> getUDFClassFromExprDesc(exprNodeDesc desc) {
+ if (!(desc instanceof exprNodeGenericFuncDesc)) {
+ return null;
+ }
+ exprNodeGenericFuncDesc genericFuncDesc = (exprNodeGenericFuncDesc)desc;
+ if (!(genericFuncDesc.getGenericUDF() instanceof GenericUDFBridge)) {
+ return null;
+ }
+ GenericUDFBridge bridge = (GenericUDFBridge)(genericFuncDesc.getGenericUDF());
+ return bridge.getUdfClass();
+ }
+
+ /**
+ * Returns whether a GenericUDF is deterministic or not.
+ */
+ public static boolean isDeterministic(GenericUDF genericUDF) {
+ UDFType genericUDFType = genericUDF.getClass().getAnnotation(UDFType.class);
+ if (genericUDFType != null && genericUDFType.deterministic() == false) {
+ return false;
+ }
+
+ if (genericUDF instanceof GenericUDFBridge) {
+ GenericUDFBridge bridge = (GenericUDFBridge)(genericUDF);
+ UDFType bridgeUDFType = bridge.getUdfClass().getAnnotation(UDFType.class);
+ if (bridgeUDFType != null && bridgeUDFType.deterministic() == false) {
+ return false;
+ }
+ }
+
+ return true;
}
+
+ /**
+ * Returns whether the exprNodeDesc is a node of "and", "or", "not".
+ */
+ public static boolean isOpAndOrNot(exprNodeDesc desc) {
+ Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
+ return UDFOPAnd.class == udfClass
+ || UDFOPOr.class == udfClass
+ || UDFOPNot.class == udfClass;
+ }
+
+ /**
+ * Returns whether the exprNodeDesc is a node of "and".
+ */
+ public static boolean isOpAnd(exprNodeDesc desc) {
+ Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
+ return UDFOPAnd.class == udfClass;
+ }
+
+ /**
+ * Returns whether the exprNodeDesc is a node of "positive".
+ */
+ public static boolean isOpPositive(exprNodeDesc desc) {
+ Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
+ return UDFOPPositive.class == udfClass;
+ }
+
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Fri Aug 14 07:48:02 2009
@@ -32,7 +32,6 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.hive.ql.exec.FunctionInfo.OperatorType;
public class FunctionTask extends Task<FunctionWork> {
private static final long serialVersionUID = 1L;
@@ -64,8 +63,7 @@
Class<?> udfClass = getUdfClass(createFunctionDesc);
if(UDF.class.isAssignableFrom(udfClass)) {
FunctionRegistry.registerTemporaryUDF(createFunctionDesc.getFunctionName(),
- (Class<? extends UDF>) udfClass,
- OperatorType.PREFIX, false);
+ (Class<? extends UDF>) udfClass, false);
return 0;
} else if(GenericUDF.class.isAssignableFrom(udfClass)) {
FunctionRegistry.registerTemporaryGenericUDF(createFunctionDesc.getFunctionName(),
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Fri Aug 14 07:48:02 2009
@@ -193,7 +193,7 @@
aggregationEvaluators = new GenericUDAFEvaluator[conf.getAggregators().size()];
for (int i = 0; i < aggregationEvaluators.length; i++) {
aggregationDesc agg = conf.getAggregators().get(i);
- aggregationEvaluators[i] = agg.createGenericUDAFEvaluator();
+ aggregationEvaluators[i] = agg.getGenericUDAFEvaluator();
}
// init objectInspectors
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/NumericOpMethodResolver.java Fri Aug 14 07:48:02 2009
@@ -80,7 +80,12 @@
for(Method m: Arrays.asList(udfClass.getMethods())) {
if (m.getName().equals("evaluate")) {
- List<TypeInfo> argumentTypeInfos = TypeInfoUtils.getParameterTypeInfos(m);
+ List<TypeInfo> argumentTypeInfos = TypeInfoUtils.getParameterTypeInfos(m,
+ pTypeInfos.size());
+ if (argumentTypeInfos == null) {
+ // null means the method does not accept number of arguments passed.
+ continue;
+ }
boolean match = (argumentTypeInfos.size() == pTypeInfos.size());
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFArgumentException.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFArgumentException.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFArgumentException.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFArgumentException.java Fri Aug 14 07:48:02 2009
@@ -32,4 +32,9 @@
public UDFArgumentException(String message) {
super(message);
}
+
+ public UDFArgumentException(Throwable cause) {
+ super(cause);
+ }
+
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java Fri Aug 14 07:48:02 2009
@@ -25,6 +25,7 @@
import java.util.Map;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -39,7 +40,6 @@
import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.UDFOPAnd;
@@ -80,22 +80,30 @@
}
/**
- * Process function descriptors.
+ * If all children are candidates and refer only to one table alias then this expr is a candidate
+ * else it is not a candidate but its children could be final candidates
*/
- public static class FuncExprProcessor implements NodeProcessor {
+ public static class GenericFuncExprProcessor implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
exprNodeDesc newfd = null;
- exprNodeFuncDesc fd = (exprNodeFuncDesc) nd;
+ exprNodeGenericFuncDesc fd = (exprNodeGenericFuncDesc) nd;
boolean unknown = false;
- // Check if any of the children is unknown for non logical operators
- if (!fd.getUDFMethod().getDeclaringClass().equals(UDFOPAnd.class)
- && !fd.getUDFMethod().getDeclaringClass().equals(UDFOPOr.class)
- && !fd.getUDFMethod().getDeclaringClass().equals(UDFOPNot.class))
+
+ if (FunctionRegistry.isOpAndOrNot(fd)) {
+ // do nothing because "And" and "Or" and "Not" supports null value evaluation
+ // NOTE: In the future all UDFs that treats null value as UNKNOWN (both in parameters and return
+ // values) should derive from a common base class UDFNullAsUnknown, so instead of listing the classes
+ // here we would test whether a class is derived from that base class.
+ } else if (!FunctionRegistry.isDeterministic(fd.getGenericUDF())) {
+ // If it's a non-deterministic UDF, set unknown to true
+ unknown = true;
+ } else {
+ // If any child is null, set unknown to true
for(Object child: nodeOutputs) {
exprNodeDesc child_nd = (exprNodeDesc)child;
if (child_nd instanceof exprNodeConstantDesc &&
@@ -103,49 +111,6 @@
unknown = true;
}
}
-
- if (fd.getUDFClass().getAnnotation(UDFType.class) != null &&
- (fd.getUDFClass().getAnnotation(UDFType.class).deterministic() == false ||
- unknown))
- newfd = new exprNodeConstantDesc(fd.getTypeInfo(), null);
- else {
- // Create the list of children
- ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>();
- for(Object child: nodeOutputs) {
- children.add((exprNodeDesc) child);
- }
- // Create a copy of the function descriptor
- newfd = new exprNodeFuncDesc(fd.getMethodName(),
- fd.getTypeInfo(), fd.getUDFClass(),
- fd.getUDFMethod(), children);
- }
-
- return newfd;
- }
-
- }
-
- /**
- * If all children are candidates and refer only to one table alias then this expr is a candidate
- * else it is not a candidate but its children could be final candidates
- */
- public static class GenericFuncExprProcessor implements NodeProcessor {
-
- @Override
- public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
- Object... nodeOutputs) throws SemanticException {
-
- exprNodeDesc newfd = null;
- exprNodeGenericFuncDesc fd = (exprNodeGenericFuncDesc) nd;
-
- boolean unknown = false;
- // Check if any of the children is unknown
- for(Object child: nodeOutputs) {
- exprNodeDesc child_nd = (exprNodeDesc)child;
- if (child_nd instanceof exprNodeConstantDesc &&
- ((exprNodeConstantDesc)child_nd).getValue() == null) {
- unknown = true;
- }
}
if (unknown)
@@ -157,7 +122,7 @@
children.add((exprNodeDesc) child);
}
// Create a copy of the function descriptor
- newfd = new exprNodeGenericFuncDesc(fd.getTypeInfo(), fd.getGenericUDFClass(), children);
+ newfd = new exprNodeGenericFuncDesc(fd.getTypeInfo(), fd.getGenericUDF(), children);
}
return newfd;
@@ -220,10 +185,6 @@
return new DefaultExprProcessor();
}
- public static NodeProcessor getFuncProcessor() {
- return new FuncExprProcessor();
- }
-
public static NodeProcessor getGenericFuncProcessor() {
return new GenericFuncExprProcessor();
}
@@ -253,7 +214,6 @@
Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
exprRules.put(new RuleRegExp("R1", exprNodeColumnDesc.class.getName() + "%"), getColumnProcessor());
exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFieldProcessor());
- exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
exprRules.put(new RuleRegExp("R5", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
// The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTPartitionPruner.java Fri Aug 14 07:48:02 2009
@@ -20,6 +20,7 @@
import java.util.*;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -29,7 +30,6 @@
import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.UDFOPAnd;
@@ -259,21 +259,14 @@
.getMsg(expr, e.getMessage()));
}
- if (desc instanceof exprNodeFuncDesc && (
- ((exprNodeFuncDesc)desc).getUDFMethod().getDeclaringClass().equals(UDFOPAnd.class)
- || ((exprNodeFuncDesc)desc).getUDFMethod().getDeclaringClass().equals(UDFOPOr.class)
- || ((exprNodeFuncDesc)desc).getUDFMethod().getDeclaringClass().equals(UDFOPNot.class))) {
+ if (FunctionRegistry.isOpAndOrNot(desc)) {
// do nothing because "And" and "Or" and "Not" supports null value evaluation
// NOTE: In the future all UDFs that treats null value as UNKNOWN (both in parameters and return
// values) should derive from a common base class UDFNullAsUnknown, so instead of listing the classes
// here we would test whether a class is derived from that base class.
} else if (mightBeUnknown(desc) ||
- (desc instanceof exprNodeFuncDesc &&
- ((exprNodeFuncDesc)desc).getUDFClass().getAnnotation(UDFType.class) != null &&
- ((exprNodeFuncDesc)desc).getUDFClass().getAnnotation(UDFType.class).deterministic() == false) ||
- (desc instanceof exprNodeGenericFuncDesc &&
- ((exprNodeGenericFuncDesc)desc).getGenericUDFClass().getAnnotation(UDFType.class) != null &&
- ((exprNodeGenericFuncDesc)desc).getGenericUDFClass().getAnnotation(UDFType.class).deterministic() == false))
+ ((desc instanceof exprNodeGenericFuncDesc) &&
+ !FunctionRegistry.isDeterministic(((exprNodeGenericFuncDesc)desc).getGenericUDF())))
{
// If its a non-deterministic UDF or if any child is null, set this node to null
LOG.trace("Pruner function might be unknown: " + expr.toStringTree());
@@ -322,14 +315,6 @@
} else if (desc instanceof exprNodeFieldDesc) {
exprNodeFieldDesc d = (exprNodeFieldDesc)desc;
return mightBeUnknown(d.getDesc());
- } else if (desc instanceof exprNodeFuncDesc) {
- exprNodeFuncDesc d = (exprNodeFuncDesc)desc;
- for(int i=0; i<d.getChildren().size(); i++) {
- if (mightBeUnknown(d.getChildExprs().get(i))) {
- return true;
- }
- }
- return false;
} else if (desc instanceof exprNodeGenericFuncDesc) {
exprNodeGenericFuncDesc d = (exprNodeGenericFuncDesc)desc;
for(int i=0; i<d.getChildren().size(); i++) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Aug 14 07:48:02 2009
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.parse;
import java.io.Serializable;
-import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Formatter;
import java.util.HashMap;
@@ -43,15 +42,21 @@
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.ExecDriver;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.MapRedTask;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.UnionOperator;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@ -82,24 +87,24 @@
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3;
import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink4;
-import org.apache.hadoop.hive.ql.plan.*;
-import org.apache.hadoop.hive.ql.exec.*;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.aggregationDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.extractDesc;
import org.apache.hadoop.hive.ql.plan.fetchWork;
import org.apache.hadoop.hive.ql.plan.fileSinkDesc;
import org.apache.hadoop.hive.ql.plan.filterDesc;
+import org.apache.hadoop.hive.ql.plan.forwardDesc;
import org.apache.hadoop.hive.ql.plan.groupByDesc;
import org.apache.hadoop.hive.ql.plan.joinDesc;
import org.apache.hadoop.hive.ql.plan.limitDesc;
import org.apache.hadoop.hive.ql.plan.loadFileDesc;
import org.apache.hadoop.hive.ql.plan.loadTableDesc;
+import org.apache.hadoop.hive.ql.plan.mapredWork;
import org.apache.hadoop.hive.ql.plan.moveWork;
import org.apache.hadoop.hive.ql.plan.partitionDesc;
import org.apache.hadoop.hive.ql.plan.reduceSinkDesc;
@@ -130,8 +135,6 @@
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.serde.Constants;
-
/**
* Implementation of the semantic analyzer
*/
@@ -2222,7 +2225,7 @@
static ArrayList<GenericUDAFEvaluator> getUDAFEvaluators(ArrayList<aggregationDesc> aggs) {
ArrayList<GenericUDAFEvaluator> result = new ArrayList<GenericUDAFEvaluator>();
for (int i=0; i<aggs.size(); i++) {
- result.add(aggs.get(i).createGenericUDAFEvaluator());
+ result.add(aggs.get(i).getGenericUDAFEvaluator());
}
return result;
}
@@ -3920,7 +3923,7 @@
exprNodeDesc equalsExpr = null;
{
exprNodeDesc hashfnExpr = new exprNodeGenericFuncDesc(TypeInfoFactory.intTypeInfo,
- GenericUDFHash.class, args);
+ new GenericUDFHash(), args);
assert(hashfnExpr != null);
LOG.info("hashfnExpr = " + hashfnExpr);
exprNodeDesc andExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr);
@@ -4514,49 +4517,6 @@
}
- public static ArrayList<exprNodeDesc> convertParameters(Method m, List<exprNodeDesc> parametersPassed) {
-
- ArrayList<exprNodeDesc> newParameters = new ArrayList<exprNodeDesc>();
- List<TypeInfo> parametersAccepted = TypeInfoUtils.getParameterTypeInfos(m);
-
- // 0 is the function name
- for (int i = 0; i < parametersPassed.size(); i++) {
- exprNodeDesc descPassed = parametersPassed.get(i);
- TypeInfo typeInfoPassed = descPassed.getTypeInfo();
- TypeInfo typeInfoAccepted = parametersAccepted.get(i);
- if (descPassed instanceof exprNodeNullDesc) {
- exprNodeConstantDesc newCh = new exprNodeConstantDesc(typeInfoAccepted, null);
- newParameters.add(newCh);
- } else if (typeInfoAccepted.equals(typeInfoPassed)
- || typeInfoAccepted.equals(TypeInfoFactory.unknownTypeInfo)
- || (typeInfoAccepted.equals(TypeInfoFactory.unknownMapTypeInfo)
- && typeInfoPassed.getCategory().equals(Category.MAP))
- || (typeInfoAccepted.equals(TypeInfoFactory.unknownListTypeInfo)
- && typeInfoPassed.getCategory().equals(Category.LIST))
- ) {
- // no type conversion needed
- newParameters.add(descPassed);
- } else {
- // must be implicit type conversion
- TypeInfo to = typeInfoAccepted;
- if (!FunctionRegistry.implicitConvertable(typeInfoPassed, to)) {
- throw new RuntimeException("Internal exception: cannot convert from " + typeInfoPassed + " to " + to);
- }
- Method conv = FunctionRegistry.getUDFMethod(to.getTypeName(), typeInfoPassed);
- assert(conv != null);
- Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getTypeName());
- assert(c != null);
-
- // get the conversion method
- ArrayList<exprNodeDesc> conversionArg = new ArrayList<exprNodeDesc>(1);
- conversionArg.add(descPassed);
- newParameters.add(new exprNodeFuncDesc(to.getTypeName(), typeInfoAccepted, c, conv, conversionArg));
- }
- }
-
- return newParameters;
- }
-
public void validate() throws SemanticException {
// Check if the plan contains atleast one path.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Fri Aug 14 07:48:02 2009
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.parse;
-import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HashMap;
@@ -27,10 +26,10 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.AmbiguousMethodException;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -41,7 +40,6 @@
import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
import org.apache.hadoop.hive.serde.Constants;
@@ -55,8 +53,6 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.util.ReflectionUtils;
/**
* The Factory for creating typecheck processors. The typecheck processors are used to
@@ -419,8 +415,9 @@
* @throws UDFArgumentException
*/
public static exprNodeDesc getFuncExprNodeDesc(String name, exprNodeDesc... children) {
+ ArrayList<exprNodeDesc> c = new ArrayList<exprNodeDesc>(Arrays.asList(children));
try {
- return getFuncExprNodeDesc(name, Arrays.asList(children));
+ return getFuncExprNodeDesc(name, c);
} catch (UDFArgumentException e) {
throw new RuntimeException("Hive 2 internal error", e);
}
@@ -429,48 +426,22 @@
/**
* This function create an ExprNodeDesc for a UDF function given the children (arguments).
* It will insert implicit type conversion functions if necessary.
- * @throws SemanticException
+ * @throws UDFArgumentException
*/
public static exprNodeDesc getFuncExprNodeDesc(String udfName, List<exprNodeDesc> children)
throws UDFArgumentException {
FunctionInfo fi = FunctionRegistry.getFunctionInfo(udfName);
- if (fi == null) return null;
-
- // Is it a generic UDF?
- Class<? extends GenericUDF> genericUDFClass = fi.getGenericUDFClass();
- if (genericUDFClass != null) {
- return exprNodeGenericFuncDesc.newInstance(genericUDFClass, children);
+ if (fi == null) {
+ throw new UDFArgumentException("udf:" + udfName + " not found.");
}
- // TODO: extract as a function
- // Find the corresponding method
- ArrayList<TypeInfo> argumentTypeInfos = new ArrayList<TypeInfo>(children.size());
- for(int i=0; i<children.size(); i++) {
- exprNodeDesc child = children.get(i);
- argumentTypeInfos.add(child.getTypeInfo());
+ GenericUDF genericUDF = fi.getGenericUDF();
+ if (genericUDF == null) {
+ throw new UDFArgumentException("udf:" + udfName + " is an aggregation function.");
}
-
- Method udfMethod = FunctionRegistry.getUDFMethod(udfName, argumentTypeInfos);
- if (udfMethod == null) return null;
- // Convert the parameters if the type of parameters do not exactly match.
- ArrayList<exprNodeDesc> ch = SemanticAnalyzer.convertParameters(udfMethod, children);
-
- // The return type of a function can be of either Java Primitive Type/Class or Writable Class.
- TypeInfo resultTypeInfo = null;
- if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(udfMethod.getReturnType())) {
- resultTypeInfo = TypeInfoFactory.getPrimitiveTypeInfoFromPrimitiveWritable(udfMethod.getReturnType());
- } else {
- resultTypeInfo = TypeInfoFactory.getPrimitiveTypeInfoFromJavaPrimitive(udfMethod.getReturnType());
- }
-
- exprNodeFuncDesc desc = new exprNodeFuncDesc(
- udfName,
- resultTypeInfo,
- FunctionRegistry.getUDFClass(udfName),
- udfMethod, ch);
- return desc;
+ return exprNodeGenericFuncDesc.newInstance(genericUDF, children);
}
static exprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction,
@@ -529,7 +500,7 @@
// Calculate TypeInfo
TypeInfo t = ((ListTypeInfo)myt).getListElementTypeInfo();
desc = new exprNodeGenericFuncDesc(t,
- FunctionRegistry.getGenericUDFClassForIndex(),
+ FunctionRegistry.getGenericUDFForIndex(),
children);
}
else if (myt.getCategory() == Category.MAP) {
@@ -544,7 +515,7 @@
// Calculate TypeInfo
TypeInfo t = ((MapTypeInfo)myt).getMapValueTypeInfo();
desc = new exprNodeGenericFuncDesc(t,
- FunctionRegistry.getGenericUDFClassForIndex(),
+ FunctionRegistry.getGenericUDFForIndex(),
children);
}
else {
@@ -562,8 +533,9 @@
throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr));
}
- desc = getFuncExprNodeDesc(funcText, children);
- if (desc == null) {
+ try {
+ desc = getFuncExprNodeDesc(funcText, children);
+ } catch (AmbiguousMethodException e) {
ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
for(int i=0; i<children.size(); i++) {
argumentClasses.add(((PrimitiveTypeInfo)children.get(i).getTypeInfo()).getPrimitiveWritableClass());
@@ -571,22 +543,19 @@
if (isFunction) {
String reason = "Looking for UDF \"" + expr.getChild(0).getText() + "\" with parameters " + argumentClasses;
- throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason));
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason), e);
} else {
String reason = "Looking for Operator \"" + expr.getText() + "\" with parameters " + argumentClasses;
- throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason));
+ throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason), e);
}
}
}
// UDFOPPositive is a no-op.
// However, we still create it, and then remove it here, to make sure we only allow
// "+" for numeric types.
- if (desc instanceof exprNodeFuncDesc) {
- exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc;
- if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) {
- assert(funcDesc.getChildren().size() == 1);
- desc = funcDesc.getChildExprs().get(0);
- }
+ if (FunctionRegistry.isOpPositive(desc)) {
+ assert(desc.getChildren().size() == 1);
+ desc = desc.getChildren().get(0);
}
assert(desc != null);
return desc;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java Fri Aug 14 07:48:02 2009
@@ -29,7 +29,14 @@
public class aggregationDesc implements java.io.Serializable {
private static final long serialVersionUID = 1L;
private String genericUDAFName;
- private Class<? extends GenericUDAFEvaluator> genericUDAFEvaluatorClass;
+
+ /**
+ * In case genericUDAFEvaluator is Serializable, we will serialize the object.
+ *
+ * In case genericUDAFEvaluator does not implement Serializable, Java will remember the
+ * class of genericUDAFEvaluator and creates a new instance when deserialized. This is
+ * exactly what we want.
+ */
private GenericUDAFEvaluator genericUDAFEvaluator;
private java.util.ArrayList<exprNodeDesc> parameters;
private boolean distinct;
@@ -42,21 +49,11 @@
final boolean distinct,
final GenericUDAFEvaluator.Mode mode) {
this.genericUDAFName = genericUDAFName;
- if (genericUDAFEvaluator instanceof Serializable) {
- this.genericUDAFEvaluator = genericUDAFEvaluator;
- this.genericUDAFEvaluatorClass = null;
- } else {
- this.genericUDAFEvaluator = null;
- this.genericUDAFEvaluatorClass = genericUDAFEvaluator.getClass();
- }
+ this.genericUDAFEvaluator = genericUDAFEvaluator;
this.parameters = parameters;
this.distinct = distinct;
this.mode = mode;
}
- public GenericUDAFEvaluator createGenericUDAFEvaluator() {
- return (genericUDAFEvaluator != null) ? genericUDAFEvaluator
- : (GenericUDAFEvaluator)ReflectionUtils.newInstance(genericUDAFEvaluatorClass, null);
- }
public void setGenericUDAFName(final String genericUDAFName) {
this.genericUDAFName = genericUDAFName;
}
@@ -69,12 +66,6 @@
public GenericUDAFEvaluator getGenericUDAFEvaluator() {
return genericUDAFEvaluator;
}
- public void setGenericUDAFEvaluatorClass(final Class<? extends GenericUDAFEvaluator> genericUDAFEvaluatorClass) {
- this.genericUDAFEvaluatorClass = genericUDAFEvaluatorClass;
- }
- public Class<? extends GenericUDAFEvaluator> getGenericUDAFEvaluatorClass() {
- return genericUDAFEvaluatorClass;
- }
public java.util.ArrayList<exprNodeDesc> getParameters() {
return this.parameters;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java Fri Aug 14 07:48:02 2009
@@ -65,7 +65,7 @@
}
@Override
- public List<? extends Node> getChildren() {
+ public List<exprNodeDesc> getChildren() {
return null;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFieldDesc.java Fri Aug 14 07:48:02 2009
@@ -24,8 +24,6 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
public class exprNodeFieldDesc extends exprNodeDesc implements Serializable {
@@ -46,8 +44,8 @@
}
@Override
- public List<? extends Node> getChildren() {
- List<Node> children = new ArrayList<Node>(2);
+ public List<exprNodeDesc> getChildren() {
+ List<exprNodeDesc> children = new ArrayList<exprNodeDesc>(2);
children.add(desc);
return children;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java Fri Aug 14 07:48:02 2009
@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.plan;
-
-import java.io.Serializable;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.exec.UDF;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.lib.Node;
-
-/**
- * The reason that we have to store UDFClass as well as UDFMethod is because
- * UDFMethod might be declared in a parent class of UDFClass. As a result,
- * UDFMethod.getDeclaringClass() may not work.
- */
-public class exprNodeFuncDesc extends exprNodeDesc implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private Class<? extends UDF> UDFClass;
- private Method UDFMethod;
- private List<exprNodeDesc> childExprs;
- private String methodName;
-
- public exprNodeFuncDesc() {}
- public exprNodeFuncDesc(String methodName, TypeInfo typeInfo, Class<? extends UDF> UDFClass,
- Method UDFMethod, List<exprNodeDesc> children) {
- super(typeInfo);
- assert(UDFClass != null);
- this.UDFClass = UDFClass;
- assert(UDFMethod != null);
- this.UDFMethod = UDFMethod;
- this.childExprs = children;
- this.methodName = methodName;
- }
-
- public Class<? extends UDF> getUDFClass() {
- return UDFClass;
- }
-
- public void setUDFClass(Class<? extends UDF> UDFClass) {
- this.UDFClass = UDFClass;
- }
- public Method getUDFMethod() {
- return this.UDFMethod;
- }
- public void setUDFMethod(Method method) {
- this.UDFMethod = method;
- }
- public List<exprNodeDesc> getChildExprs() {
- return this.childExprs;
- }
- public void setChildExprs(List<exprNodeDesc> children) {
- this.childExprs = children;
- }
- @Override
- public List<? extends Node> getChildren() {
- return (List<? extends Node>)this.childExprs;
- }
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(UDFClass.toString());
- sb.append(".");
- sb.append(UDFMethod.toString());
- sb.append("(");
- for(int i=0; i<childExprs.size(); i++) {
- if (i>0) sb.append(", ");
- sb.append(childExprs.get(i).toString());
- }
- sb.append("(");
- sb.append(")");
- return sb.toString();
- }
-
- @explain(displayName="expr")
- @Override
- public String getExprString() {
- FunctionInfo fI = FunctionRegistry.getFunctionInfo(methodName);
- StringBuilder sb = new StringBuilder();
-
- if (fI.getOpType() == FunctionInfo.OperatorType.PREFIX ||
- fI.isAggFunction()) {
- sb.append(fI.getDisplayName());
- if (!fI.isOperator()) {
- sb.append("(");
- }
- else {
- sb.append(" ");
- }
-
- boolean first = true;
- for(exprNodeDesc chld: childExprs) {
- if (!first) {
- sb.append(", ");
- }
- first = false;
-
- sb.append(chld.getExprString());
- }
-
- if(!fI.isOperator()) {
- sb.append(")");
- }
- }
- else if (fI.getOpType() == FunctionInfo.OperatorType.INFIX) {
- // assert that this has only 2 children
- assert(childExprs.size() == 2);
- sb.append("(");
- sb.append(childExprs.get(0).getExprString());
- sb.append(" ");
- sb.append(fI.getDisplayName());
- sb.append(" ");
- sb.append(childExprs.get(1).getExprString());
- sb.append(")");
- }
- else if (fI.getOpType() == FunctionInfo.OperatorType.POSTFIX) {
- // assert for now as there should be no such case
- assert(childExprs.size() == 1);
- sb.append(childExprs.get(0).getExprString());
- sb.append(" ");
- sb.append(fI.getDisplayName());
- }
-
- return sb.toString();
- }
-
- public List<String> getCols() {
- List<String> colList = new ArrayList<String>();
- if (childExprs != null) {
- int pos = 0;
- while (pos < childExprs.size()) {
- List<String> colCh = childExprs.get(pos).getCols();
- colList = Utilities.mergeUniqElems(colList, colCh);
- pos++;
- }
- }
-
- return colList;
- }
-
- public String getMethodName() {
- return methodName;
- }
- public void setMethodName(String methodName) {
- this.methodName = methodName.toLowerCase();
- }
- @Override
- public exprNodeDesc clone() {
- List<exprNodeDesc> cloneCh = new ArrayList<exprNodeDesc>(childExprs.size());
- for(exprNodeDesc ch : childExprs) {
- cloneCh.add(ch.clone());
- }
- exprNodeFuncDesc clone = new exprNodeFuncDesc(methodName, this.typeInfo,
- this.UDFClass, this.UDFMethod, cloneCh);
- return clone;
- }
-
- @Override
- public boolean isSame(Object o) {
- if (!(o instanceof exprNodeFuncDesc))
- return false;
- exprNodeFuncDesc dest = (exprNodeFuncDesc)o;
- if (!typeInfo.equals(dest.getTypeInfo()) ||
- !UDFClass.equals(dest.getUDFClass()) ||
- !UDFMethod.equals(dest.getUDFMethod()))
- return false;
-
- if (childExprs.size() != dest.getChildExprs().size())
- return false;
-
- for (int pos = 0; pos < childExprs.size(); pos++) {
- if (!childExprs.get(pos).isSame(dest.getChildExprs().get(pos)))
- return false;
- }
-
- return true;
- }
-}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeGenericFuncDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeGenericFuncDesc.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeGenericFuncDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeGenericFuncDesc.java Fri Aug 14 07:48:02 2009
@@ -25,11 +25,11 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
/**
* Describes a GenericFunc node.
@@ -37,24 +37,32 @@
public class exprNodeGenericFuncDesc extends exprNodeDesc implements Serializable {
private static final long serialVersionUID = 1L;
- private Class<? extends GenericUDF> genericUDFClass;
+
+ /**
+ * In case genericUDF is Serializable, we will serialize the object.
+ *
+ * In case genericUDF does not implement Serializable, Java will remember the
+ * class of genericUDF and creates a new instance when deserialized. This is
+ * exactly what we want.
+ */
+ private GenericUDF genericUDF;
private List<exprNodeDesc> childExprs;
public exprNodeGenericFuncDesc() {}
- public exprNodeGenericFuncDesc(TypeInfo typeInfo, Class<? extends GenericUDF> genericUDFClass,
+ public exprNodeGenericFuncDesc(TypeInfo typeInfo, GenericUDF genericUDF,
List<exprNodeDesc> children) {
super(typeInfo);
- assert(genericUDFClass != null);
- this.genericUDFClass = genericUDFClass;
+ assert(genericUDF != null);
+ this.genericUDF = genericUDF;
this.childExprs = children;
}
- public Class<? extends GenericUDF> getGenericUDFClass() {
- return genericUDFClass;
+ public GenericUDF getGenericUDF() {
+ return genericUDF;
}
- public void setGenericUDFClass(Class<? extends GenericUDF> GenericUDFClass) {
- this.genericUDFClass = GenericUDFClass;
+ public void setGenericUDF(GenericUDF genericUDF) {
+ this.genericUDF = genericUDF;
}
public List<exprNodeDesc> getChildExprs() {
@@ -64,12 +72,12 @@
this.childExprs = children;
}
@Override
- public List<? extends Node> getChildren() {
- return (List<? extends Node>)this.childExprs;
+ public List<exprNodeDesc> getChildren() {
+ return childExprs;
}
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append(genericUDFClass.toString());
+ sb.append(genericUDF.getClass().toString());
sb.append("(");
for(int i=0; i<childExprs.size(); i++) {
if (i>0) sb.append(", ");
@@ -89,7 +97,6 @@
childrenExprStrings[i] = childExprs.get(i).getExprString();
}
- GenericUDF genericUDF = (GenericUDF) ReflectionUtils.newInstance(genericUDFClass, null);
return genericUDF.getDisplayString(childrenExprStrings);
}
@@ -114,7 +121,7 @@
cloneCh.add(ch.clone());
}
exprNodeGenericFuncDesc clone = new exprNodeGenericFuncDesc(this.typeInfo,
- this.genericUDFClass, cloneCh);
+ FunctionRegistry.cloneGenericUDF(genericUDF), cloneCh);
return clone;
}
@@ -123,18 +130,17 @@
* parameters.
* @throws UDFArgumentException
*/
- public static exprNodeGenericFuncDesc newInstance(Class<? extends GenericUDF> genericUDFClass,
+ public static exprNodeGenericFuncDesc newInstance(GenericUDF genericUDF,
List<exprNodeDesc> children) throws UDFArgumentException {
ObjectInspector[] childrenOIs = new ObjectInspector[children.size()];
for(int i=0; i<childrenOIs.length; i++) {
childrenOIs[i] = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(
children.get(i).getTypeInfo());
}
- GenericUDF genericUDF = (GenericUDF) ReflectionUtils.newInstance(genericUDFClass, null);
ObjectInspector oi = genericUDF.initialize(childrenOIs);
return new exprNodeGenericFuncDesc(TypeInfoUtils.getTypeInfoFromObjectInspector(oi),
- genericUDFClass, children);
+ genericUDF, children);
}
@Override
@@ -143,9 +149,18 @@
return false;
exprNodeGenericFuncDesc dest = (exprNodeGenericFuncDesc)o;
if (!typeInfo.equals(dest.getTypeInfo()) ||
- !genericUDFClass.equals(dest.getGenericUDFClass()))
+ !genericUDF.getClass().equals(dest.getGenericUDF().getClass()))
return false;
+ if (genericUDF instanceof GenericUDFBridge) {
+ GenericUDFBridge bridge = (GenericUDFBridge) genericUDF;
+ GenericUDFBridge bridge2 = (GenericUDFBridge) dest.getGenericUDF();
+ if (!bridge.getUdfClass().equals(bridge2.getUdfClass())
+ || !bridge.getUdfName().equals(bridge2.getUdfName())
+ || bridge.isOperator() != bridge2.isOperator())
+ return false;
+ }
+
if (childExprs.size() != dest.getChildExprs().size())
return false;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java Fri Aug 14 07:48:02 2009
@@ -24,6 +24,7 @@
import java.util.Map;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
@@ -39,7 +40,6 @@
import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.UDFOPAnd;
import org.apache.hadoop.hive.ql.udf.UDFType;
@@ -88,57 +88,6 @@
}
- /**
- * If all children are candidates and refer only to one table alias then this expr is a candidate
- * else it is not a candidate but its children could be final candidates
- */
- public static class FuncExprProcessor implements NodeProcessor {
-
- @Override
- public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
- Object... nodeOutputs) throws SemanticException {
- ExprWalkerInfo ctx = (ExprWalkerInfo) procCtx;
- String alias = null;
- exprNodeFuncDesc expr = (exprNodeFuncDesc) nd;
-
- UDFType note = expr.getUDFClass().getAnnotation(UDFType.class);
- if(note != null && !note.deterministic()) {
- // this UDF can't be pushed down
- ctx.setIsCandidate(expr, false);
- ctx.setDeterministic(false);
- return false;
- }
-
- boolean isCandidate = true;
- for (int i=0; i < nd.getChildren().size(); i++) {
- exprNodeDesc ch = (exprNodeDesc) nd.getChildren().get(i);
- exprNodeDesc newCh = ctx.getConvertedNode(ch);
- if (newCh != null) {
- expr.getChildExprs().set(i, newCh);
- ch = newCh;
- }
- String chAlias = ctx.getAlias(ch);
-
- isCandidate = isCandidate && ctx.isCandidate(ch);
- // need to iterate through all children even if one is found to be not a candidate
- // in case if the other children could be individually pushed up
- if (isCandidate && chAlias != null) {
- if (alias == null) {
- alias = chAlias;
- } else if (!chAlias.equalsIgnoreCase(alias)) {
- isCandidate = false;
- }
- }
-
- if(!isCandidate)
- break;
- }
- ctx.addAlias(expr, alias);
- ctx.setIsCandidate(expr, isCandidate);
- return isCandidate;
- }
-
- }
public static class FieldExprProcessor implements NodeProcessor {
@@ -190,8 +139,8 @@
String alias = null;
exprNodeGenericFuncDesc expr = (exprNodeGenericFuncDesc) nd;
- UDFType note = expr.getGenericUDFClass().getAnnotation(UDFType.class);
- if(note != null && !note.deterministic()) {
+
+ if (!FunctionRegistry.isDeterministic(expr.getGenericUDF())) {
// this GenericUDF can't be pushed down
ctx.setIsCandidate(expr, false);
ctx.setDeterministic(false);
@@ -247,10 +196,6 @@
return new DefaultExprProcessor();
}
- public static NodeProcessor getFuncProcessor() {
- return new FuncExprProcessor();
- }
-
public static NodeProcessor getGenericFuncProcessor() {
return new GenericFuncExprProcessor();
}
@@ -290,8 +235,7 @@
Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
exprRules.put(new RuleRegExp("R1", exprNodeColumnDesc.class.getName() + "%"), getColumnProcessor());
exprRules.put(new RuleRegExp("R2", exprNodeFieldDesc.class.getName() + "%"), getFieldProcessor());
- exprRules.put(new RuleRegExp("R3", exprNodeFuncDesc.class.getName() + "%"), getFuncProcessor());
- exprRules.put(new RuleRegExp("R4", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
+ exprRules.put(new RuleRegExp("R3", exprNodeGenericFuncDesc.class.getName() + "%"), getGenericFuncProcessor());
// The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), exprRules, exprContext);
@@ -322,13 +266,11 @@
return;
}
- // If the operator is AND, we can try to push down its children
- if (expr instanceof exprNodeFuncDesc
- && ((exprNodeFuncDesc)expr).getUDFClass().equals(UDFOPAnd.class)) {
- // now determine if any of the children are final candidates
+ if (FunctionRegistry.isOpAnd(expr)) {
+ // If the operator is AND, we need to determine if any of the children are final candidates.
for (Node ch : expr.getChildren()) {
extractFinalCandidates((exprNodeDesc) ch, ctx);
- }
+ }
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Fri Aug 14 07:48:02 2009
@@ -43,7 +43,7 @@
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.filterDesc;
import org.apache.hadoop.hive.ql.plan.joinCond;
import org.apache.hadoop.hive.ql.plan.joinDesc;
@@ -119,14 +119,11 @@
List<exprNodeDesc> children = new ArrayList<exprNodeDesc>(2);
children.add(condn);
children.add((exprNodeDesc) preds.get(i));
- condn = new exprNodeFuncDesc(
- "AND",
+ condn = new exprNodeGenericFuncDesc(
TypeInfoFactory.booleanTypeInfo,
- FunctionRegistry.getUDFClass("AND"),
- FunctionRegistry.getUDFMethod("AND",
- TypeInfoFactory.booleanTypeInfo,
- TypeInfoFactory.booleanTypeInfo),
- children);
+ FunctionRegistry.getGenericUDFForAnd(),
+ children
+ );
}
}
if(condn == null)
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java Fri Aug 14 07:48:02 2009
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.udf;
-
-import org.apache.hadoop.hive.ql.exec.NumericUDAF;
-import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.io.Text;
-
-
-
-public class UDAFAvg extends NumericUDAF {
-
- public static class UDAFAvgEvaluator implements UDAFEvaluator {
- private long mCount;
- private double mSum;
-
- public UDAFAvgEvaluator() {
- super();
- init();
- }
-
- public void init() {
- mSum = 0;
- mCount = 0;
- }
-
- public boolean iterate(DoubleWritable o) {
- if (o != null) {
- mSum += o.get();
- mCount ++;
- }
- return true;
- }
-
- public Text terminatePartial() {
- // This is SQL standard - average of zero items should be null.
- return mCount == 0 ? null : new Text(String.valueOf(mSum) + '/' + String.valueOf(mCount));
- }
-
- public boolean merge(Text o) {
- if (o != null) {
- String s = o.toString();
- int pos = s.indexOf('/');
- assert(pos != -1);
- mSum += Double.parseDouble(s.substring(0, pos));
- mCount += Long.parseLong(s.substring(pos+1));
- }
- return true;
- }
-
- public DoubleWritable terminate() {
- // This is SQL standard - average of zero items should be null.
- return mCount == 0 ? null : new DoubleWritable(mSum / mCount);
- }
- }
-
-}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBridge.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBridge.java?rev=804106&r1=804105&r2=804106&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBridge.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBridge.java Fri Aug 14 07:48:02 2009
@@ -19,6 +19,7 @@
import java.io.Serializable;
import java.lang.reflect.Method;
+import java.lang.reflect.Type;
import java.util.Arrays;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -27,8 +28,10 @@
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.PrimitiveConversionHelper;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ConversionHelper;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -89,7 +92,7 @@
transient Method terminatePartialMethod;
transient Method terminateMethod;
- transient PrimitiveConversionHelper conversionHelper;
+ transient ConversionHelper conversionHelper;
@Override
public ObjectInspector init(Mode m, ObjectInspector[] parameters)
@@ -120,7 +123,7 @@
} else {
aggregateMethod = mergeMethod;
}
- conversionHelper = new PrimitiveConversionHelper(aggregateMethod, parameters);
+ conversionHelper = new ConversionHelper(aggregateMethod, parameters);
// Output: get the evaluate method
Method evaluateMethod = null;
@@ -130,10 +133,10 @@
evaluateMethod = terminateMethod;
}
// Get the output ObjectInspector from the return type.
- Class<?> returnType = evaluateMethod.getReturnType();
+ Type returnType = evaluateMethod.getGenericReturnType();
try {
- return PrimitiveObjectInspectorFactory
- .getPrimitiveObjectInspectorFromClass(returnType);
+ return ObjectInspectorFactory.getReflectionObjectInspector(returnType,
+ ObjectInspectorOptions.JAVA);
} catch (RuntimeException e) {
throw new HiveException("Cannot recognize return type " + returnType +
" from " + evaluateMethod, e);