You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rh...@apache.org on 2014/01/17 06:27:42 UTC
svn commit: r1559012 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/
test/queries/clientnegative/ test/queries/clientpositive/
test/results/clientnegative/ test/results/clientpositive/
Author: rhbutani
Date: Fri Jan 17 05:27:41 2014
New Revision: 1559012
URL: http://svn.apache.org/r1559012
Log:
HIVE-6208 user-defined aggregate functions cannot be used as windowing function (Jason Dere via Harish Butani)
Added:
hive/trunk/ql/src/test/queries/clientnegative/windowing_invalid_udaf.q
hive/trunk/ql/src/test/queries/clientpositive/windowing_udaf2.q
hive/trunk/ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out
hive/trunk/ql/src/test/results/clientpositive/windowing_udaf2.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1559012&r1=1559011&r2=1559012&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Fri Jan 17 05:27:41 2014
@@ -420,7 +420,6 @@ public final class FunctionRegistry {
registerGenericUDF(true, LEAD_FUNC_NAME, GenericUDFLead.class);
registerGenericUDF(true, LAG_FUNC_NAME, GenericUDFLag.class);
- registerHiveUDAFsAsWindowFunctions();
registerWindowFunction("row_number", new GenericUDAFRowNumber());
registerWindowFunction("rank", new GenericUDAFRank());
registerWindowFunction("dense_rank", new GenericUDAFDenseRank());
@@ -1003,6 +1002,10 @@ public final class FunctionRegistry {
GenericUDAFResolver genericUDAFResolver) {
FunctionInfo fi = new FunctionInfo(isNative, functionName.toLowerCase(), genericUDAFResolver);
mFunctions.put(functionName.toLowerCase(), fi);
+
+ // All aggregate functions should also be usable as window functions
+ addFunctionInfoToWindowFunctions(functionName, fi);
+
registerNativeStatus(fi);
}
@@ -1021,6 +1024,10 @@ public final class FunctionRegistry {
functionName.toLowerCase(), new GenericUDAFBridge(
(UDAF) ReflectionUtils.newInstance(udafClass, null)));
mFunctions.put(functionName.toLowerCase(), fi);
+
+ // All aggregate functions should also be usable as window functions
+ addFunctionInfoToWindowFunctions(functionName, fi);
+
registerNativeStatus(fi);
}
@@ -1676,16 +1683,14 @@ public final class FunctionRegistry {
{
FunctionInfo fInfo = null;
if (registerAsUDAF) {
+ // Just register the function normally, will also get added to window functions.
registerGenericUDAF(true, name, wFn);
- fInfo = getFunctionInfo(name);
}
else {
- fInfo = new FunctionInfo(true,
- name.toLowerCase(), wFn);
+ name = name.toLowerCase();
+ fInfo = new FunctionInfo(true, name, wFn);
+ addFunctionInfoToWindowFunctions(name, fInfo);
}
-
- WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo);
- windowFunctions.put(name.toLowerCase(), wInfo);
}
public static WindowFunctionInfo getWindowFunctionInfo(String name)
@@ -1719,18 +1724,11 @@ public final class FunctionRegistry {
return false;
}
- static void registerHiveUDAFsAsWindowFunctions()
- {
- Set<String> fNames = getFunctionNames();
- for(String fName : fNames)
- {
- FunctionInfo fInfo = getFunctionInfo(fName);
- if ( fInfo.isGenericUDAF())
- {
- WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo);
- windowFunctions.put(fName, wInfo);
- }
- }
+ static private void addFunctionInfoToWindowFunctions(String functionName,
+ FunctionInfo functionInfo) {
+ // Assumes that the caller has already verified that functionInfo is for an aggregate function
+ WindowFunctionInfo wInfo = new WindowFunctionInfo(functionInfo);
+ windowFunctions.put(functionName.toLowerCase(), wInfo);
}
public static boolean isTableFunction(String name)
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1559012&r1=1559011&r2=1559012&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Fri Jan 17 05:27:41 2014
@@ -345,6 +345,9 @@ public class PTFTranslator {
private WindowFunctionDef translate(WindowTableFunctionDef wdwTFnDef,
WindowFunctionSpec spec) throws SemanticException {
WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(spec.getName());
+ if (wFnInfo == null) {
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(spec.getName()));
+ }
WindowFunctionDef def = new WindowFunctionDef();
def.setName(spec.getName());
def.setAlias(spec.getAlias());
Added: hive/trunk/ql/src/test/queries/clientnegative/windowing_invalid_udaf.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/windowing_invalid_udaf.q?rev=1559012&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/windowing_invalid_udaf.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/windowing_invalid_udaf.q Fri Jan 17 05:27:41 2014
@@ -0,0 +1 @@
+select nonexistfunc(key) over () from src limit 1;
Added: hive/trunk/ql/src/test/queries/clientpositive/windowing_udaf2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/windowing_udaf2.q?rev=1559012&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/windowing_udaf2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/windowing_udaf2.q Fri Jan 17 05:27:41 2014
@@ -0,0 +1,4 @@
+-- user-added aggregates should be usable as windowing functions
+create temporary function mysum as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum';
+
+select sum(key) over (), mysum(key) over () from src limit 1;
Added: hive/trunk/ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out?rev=1559012&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out Fri Jan 17 05:27:41 2014
@@ -0,0 +1,2 @@
+FAILED: SemanticException Failed to breakup Windowing invocations into Groups. At least 1 group must only depend on input columns. Also check for circular dependencies.
+Underlying error: Invalid function nonexistfunc
Added: hive/trunk/ql/src/test/results/clientpositive/windowing_udaf2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/windowing_udaf2.q.out?rev=1559012&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/windowing_udaf2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/windowing_udaf2.q.out Fri Jan 17 05:27:41 2014
@@ -0,0 +1,15 @@
+PREHOOK: query: -- user-added aggregates should be usable as windowing functions
+create temporary function mysum as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: -- user-added aggregates should be usable as windowing functions
+create temporary function mysum as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum'
+POSTHOOK: type: CREATEFUNCTION
+PREHOOK: query: select sum(key) over (), mysum(key) over () from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(key) over (), mysum(key) over () from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+130091.0 130091.0