You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/04/28 22:42:38 UTC
svn commit: r1476832 - in /hive/branches/branch-0.11/ql/src:
java/org/apache/hadoop/hive/ql/exec/ test/queries/clientpositive/
test/results/clientpositive/
Author: hashutosh
Date: Sun Apr 28 20:42:25 2013
New Revision: 1476832
URL: http://svn.apache.org/r1476832
Log:
HIVE-4019 : Ability to create and drop temporary partition function (Brock Noland via Ashutosh Chauhan)
Added:
hive/branches/branch-0.11/ql/src/test/queries/clientpositive/ptf_register_tblfn.q
hive/branches/branch-0.11/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out
Removed:
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFFunctionInfo.java
Modified:
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/branches/branch-0.11/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java?rev=1476832&r1=1476831&r2=1476832&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java Sun Apr 28 20:42:25 2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver;
+import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction;
/**
* FunctionInfo.
@@ -32,6 +34,8 @@ public class FunctionInfo {
private final boolean isNative;
+ private final boolean isInternalTableFunction;
+
private final String displayName;
private GenericUDF genericUDF;
@@ -40,11 +44,14 @@ public class FunctionInfo {
private GenericUDAFResolver genericUDAFResolver;
+ private Class<? extends TableFunctionResolver> tableFunctionResolver;
+
public FunctionInfo(boolean isNative, String displayName,
GenericUDF genericUDF) {
this.isNative = isNative;
this.displayName = displayName;
this.genericUDF = genericUDF;
+ this.isInternalTableFunction = false;
}
public FunctionInfo(boolean isNative, String displayName,
@@ -52,6 +59,7 @@ public class FunctionInfo {
this.isNative = isNative;
this.displayName = displayName;
this.genericUDAFResolver = genericUDAFResolver;
+ this.isInternalTableFunction = false;
}
public FunctionInfo(boolean isNative, String displayName,
@@ -59,6 +67,16 @@ public class FunctionInfo {
this.isNative = isNative;
this.displayName = displayName;
this.genericUDTF = genericUDTF;
+ this.isInternalTableFunction = false;
+ }
+
+ public FunctionInfo(String displayName, Class<? extends TableFunctionResolver> tFnCls)
+ {
+ this.displayName = displayName;
+ this.tableFunctionResolver = tFnCls;
+ PartitionTableFunctionDescription def = tableFunctionResolver.getAnnotation(PartitionTableFunctionDescription.class);
+ this.isNative = (def == null) ? false : def.isInternal();
+ this.isInternalTableFunction = isNative;
}
/**
@@ -90,6 +108,8 @@ public class FunctionInfo {
return genericUDAFResolver;
}
+
+
/**
* Get the Class of the UDF.
*/
@@ -109,6 +129,9 @@ public class FunctionInfo {
} else if (isGenericUDTF()) {
return genericUDTF.getClass();
}
+ if(isTableFunction()) {
+ return this.tableFunctionResolver;
+ }
return null;
}
@@ -131,6 +154,14 @@ public class FunctionInfo {
}
/**
+ * Internal table functions cannot be used in the language.
+ * {@link WindowingTableFunction}
+ */
+ public boolean isInternalTableFunction() {
+ return isInternalTableFunction;
+ }
+
+ /**
* @return TRUE if the function is a GenericUDF
*/
public boolean isGenericUDF() {
@@ -150,4 +181,11 @@ public class FunctionInfo {
public boolean isGenericUDTF() {
return null != genericUDTF;
}
+
+ /**
+ * @return TRUE if the function is a Table Function
+ */
+ public boolean isTableFunction() {
+ return null != tableFunctionResolver;
+ }
}
Modified: hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1476832&r1=1476831&r2=1476832&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/branch-0.11/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Sun Apr 28 20:42:25 2013
@@ -182,7 +182,6 @@ public final class FunctionRegistry {
public static final String NOOP_TABLE_FUNCTION = "noop";
public static final String NOOP_MAP_TABLE_FUNCTION = "noopwithmap";
- static Map<String, PTFFunctionInfo> tableFunctions = Collections.synchronizedMap(new LinkedHashMap<String, PTFFunctionInfo>());
static Map<String, WindowFunctionInfo> windowFunctions = Collections.synchronizedMap(new LinkedHashMap<String, WindowFunctionInfo>());
/*
@@ -1294,6 +1293,9 @@ public final class FunctionRegistry {
FunctionRegistry.registerTemporaryGenericUDAF(
functionName, (GenericUDAFResolver)
ReflectionUtils.newInstance(udfClass, null));
+ } else if(TableFunctionResolver.class.isAssignableFrom(udfClass)) {
+ FunctionRegistry.registerTableFunction(
+ functionName, (Class<? extends TableFunctionResolver>)udfClass);
} else {
return false;
}
@@ -1406,14 +1408,17 @@ public final class FunctionRegistry {
public static boolean isTableFunction(String name)
{
- PTFFunctionInfo tFInfo = tableFunctions.get(name.toLowerCase());
- return tFInfo != null && !tFInfo.isInternal();
+ FunctionInfo tFInfo = mFunctions.get(name.toLowerCase());
+ return tFInfo != null && !tFInfo.isInternalTableFunction() && tFInfo.isTableFunction();
}
public static TableFunctionResolver getTableFunctionResolver(String name)
{
- PTFFunctionInfo tfInfo = tableFunctions.get(name.toLowerCase());
- return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionResolver(), null);
+ FunctionInfo tfInfo = mFunctions.get(name.toLowerCase());
+ if(tfInfo.isTableFunction()) {
+ return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionClass(), null);
+ }
+ return null;
}
public static TableFunctionResolver getWindowingTableFunction()
@@ -1428,8 +1433,8 @@ public final class FunctionRegistry {
public static void registerTableFunction(String name, Class<? extends TableFunctionResolver> tFnCls)
{
- PTFFunctionInfo tInfo = new PTFFunctionInfo(name, tFnCls);
- tableFunctions.put(name.toLowerCase(), tInfo);
+ FunctionInfo tInfo = new FunctionInfo(name, tFnCls);
+ mFunctions.put(name.toLowerCase(), tInfo);
}
}
Added: hive/branches/branch-0.11/ql/src/test/queries/clientpositive/ptf_register_tblfn.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/queries/clientpositive/ptf_register_tblfn.q?rev=1476832&view=auto
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/queries/clientpositive/ptf_register_tblfn.q (added)
+++ hive/branches/branch-0.11/ql/src/test/queries/clientpositive/ptf_register_tblfn.q Sun Apr 28 20:42:25 2013
@@ -0,0 +1,29 @@
+DROP TABLE flights_tiny;
+
+create table flights_tiny (
+ORIGIN_CITY_NAME string,
+DEST_CITY_NAME string,
+YEAR int,
+MONTH int,
+DAY_OF_MONTH int,
+ARR_DELAY float,
+FL_NUM string
+);
+
+LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny;
+
+create temporary function npathtest as 'org.apache.hadoop.hive.ql.udf.ptf.NPath$NPathResolver';
+
+
+-- 1. basic Npath test
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath
+from npathtest(on
+ flights_tiny
+ distribute by fl_num
+ sort by year, month, day_of_month
+ arg1('LATE.LATE+'),
+ arg2('LATE'), arg3(arr_delay > 15),
+ arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath')
+ );
+
+drop temporary function npathtest;
Added: hive/branches/branch-0.11/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out?rev=1476832&view=auto
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out (added)
+++ hive/branches/branch-0.11/ql/src/test/results/clientpositive/ptf_register_tblfn.q.out Sun Apr 28 20:42:25 2013
@@ -0,0 +1,81 @@
+PREHOOK: query: DROP TABLE flights_tiny
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE flights_tiny
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table flights_tiny (
+ORIGIN_CITY_NAME string,
+DEST_CITY_NAME string,
+YEAR int,
+MONTH int,
+DAY_OF_MONTH int,
+ARR_DELAY float,
+FL_NUM string
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table flights_tiny (
+ORIGIN_CITY_NAME string,
+DEST_CITY_NAME string,
+YEAR int,
+MONTH int,
+DAY_OF_MONTH int,
+ARR_DELAY float,
+FL_NUM string
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@flights_tiny
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny
+PREHOOK: type: LOAD
+PREHOOK: Output: default@flights_tiny
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@flights_tiny
+PREHOOK: query: create temporary function npathtest as 'org.apache.hadoop.hive.ql.udf.ptf.NPath$NPathResolver'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: create temporary function npathtest as 'org.apache.hadoop.hive.ql.udf.ptf.NPath$NPathResolver'
+POSTHOOK: type: CREATEFUNCTION
+PREHOOK: query: -- 1. basic Npath test
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath
+from npathtest(on
+ flights_tiny
+ distribute by fl_num
+ sort by year, month, day_of_month
+ arg1('LATE.LATE+'),
+ arg2('LATE'), arg3(arr_delay > 15),
+ arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath')
+ )
+PREHOOK: type: QUERY
+PREHOOK: Input: default@flights_tiny
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1. basic Npath test
+select origin_city_name, fl_num, year, month, day_of_month, sz, tpath
+from npathtest(on
+ flights_tiny
+ distribute by fl_num
+ sort by year, month, day_of_month
+ arg1('LATE.LATE+'),
+ arg2('LATE'), arg3(arr_delay > 15),
+ arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath')
+ )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@flights_tiny
+#### A masked pattern was here ####
+Baltimore 1142 2010 10 20 6 20
+Baltimore 1142 2010 10 21 5 21
+Baltimore 1142 2010 10 22 4 22
+Baltimore 1142 2010 10 25 3 25
+Baltimore 1142 2010 10 26 2 26
+Chicago 1531 2010 10 21 2 21
+Chicago 1531 2010 10 25 3 25
+Chicago 1531 2010 10 26 2 26
+Baltimore 1599 2010 10 21 2 21
+Baltimore 1599 2010 10 25 3 25
+Baltimore 1599 2010 10 26 2 26
+Chicago 361 2010 10 20 2 20
+Washington 7291 2010 10 27 2 27
+Chicago 897 2010 10 20 4 20
+Chicago 897 2010 10 21 3 21
+Chicago 897 2010 10 22 2 22
+PREHOOK: query: drop temporary function npathtest
+PREHOOK: type: DROPFUNCTION
+POSTHOOK: query: drop temporary function npathtest
+POSTHOOK: type: DROPFUNCTION
Modified: hive/branches/branch-0.11/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.11/ql/src/test/results/clientpositive/show_functions.q.out?rev=1476832&r1=1476831&r2=1476832&view=diff
==============================================================================
--- hive/branches/branch-0.11/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/branches/branch-0.11/ql/src/test/results/clientpositive/show_functions.q.out Sun Apr 28 20:42:25 2013
@@ -110,7 +110,10 @@ month
named_struct
negative
ngrams
+noop
+noopwithmap
not
+npath
ntile
nvl
or
@@ -174,6 +177,7 @@ var_samp
variance
weekofyear
when
+windowingtablefunction
xpath
xpath_boolean
xpath_double