You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/07/31 07:34:04 UTC
svn commit: r1367411 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/udf/generic/ test/queries/clientpositive/
test/results/clientpositive/
Author: namit
Date: Tue Jul 31 05:34:03 2012
New Revision: 1367411
URL: http://svn.apache.org/viewvc?rev=1367411&view=rev
Log:
HIVE-3238 A table generating, table generating function
(Edward via namit)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java
hive/trunk/ql/src/test/queries/clientpositive/udf_inline.q
hive/trunk/ql/src/test/results/clientpositive/udf_inline.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1367411&r1=1367410&r2=1367411&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Jul 31 05:34:03 2012
@@ -204,6 +204,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFJSONTuple;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFParseUrlTuple;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFStack;
@@ -467,6 +468,7 @@ public final class FunctionRegistry {
// Generic UDTF's
registerGenericUDTF("explode", GenericUDTFExplode.class);
+ registerGenericUDTF("inline", GenericUDTFInline.class);
registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class);
registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class);
registerGenericUDTF("stack", GenericUDTFStack.class);
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java?rev=1367411&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java Tue Jul 31 05:34:03 2012
@@ -0,0 +1,76 @@
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.util.List;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+@Description(name ="inline", value= "_FUNC_( ARRAY( STRUCT()[,STRUCT()] "
++ "- explodes and array and struct into a table")
+public class GenericUDTFInline extends GenericUDTF {
+
+ private Object[] forwardObj;
+ private ListObjectInspector li;
+ private StructObjectInspector daStruct;
+
+ public GenericUDTFInline(){
+
+ }
+
+ @Override
+ public StructObjectInspector initialize(ObjectInspector[] ois) throws UDFArgumentException {
+ //There should be one argument that is a array of struct
+ if (ois.length!=1){
+ throw new UDFArgumentException("UDF tables only one argument");
+ }
+ if (ois[0].getCategory()!= Category.LIST){
+ throw new UDFArgumentException("Top level object must be an array but "
+ + "was "+ois[0].getTypeName());
+ }
+ li = (ListObjectInspector) ois[0];
+ ObjectInspector sub=li.getListElementObjectInspector();
+ if (sub.getCategory() != Category.STRUCT){
+ throw new UDFArgumentException("The sub element must be struct, but was "+sub.getTypeName());
+ }
+ daStruct = (StructObjectInspector) sub;
+ forwardObj = new Object[daStruct.getAllStructFieldRefs().size()];
+ return daStruct;
+ }
+
+ @Override
+ public void process(Object[] os) throws HiveException {
+ //list is always one item
+ List l = li.getList(os);
+ List<? extends StructField> fields = this.daStruct.getAllStructFieldRefs();
+ for (Object linner: l ){
+ List<List> innerList = (List) linner;
+ for (List rowList : innerList){
+ int i=0;
+ for (StructField f: fields){
+ GenericUDFUtils.ReturnObjectInspectorResolver res
+ = new GenericUDFUtils.ReturnObjectInspectorResolver();
+ res.update(f.getFieldObjectInspector());
+ this.forwardObj[i]=res.convertIfNecessary(rowList.get(i), f.getFieldObjectInspector());
+ i++;
+ }
+ forward(this.forwardObj);
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HiveException {
+ }
+
+ @Override
+ public String toString() {
+ return "inline";
+ }
+}
Added: hive/trunk/ql/src/test/queries/clientpositive/udf_inline.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udf_inline.q?rev=1367411&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udf_inline.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/udf_inline.q Tue Jul 31 05:34:03 2012
@@ -0,0 +1,18 @@
+describe function inline;
+
+explain SELECT inline(
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ )
+) as (id, text) FROM SRC limit 2;
+
+SELECT inline(
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ )
+) as (id, text) FROM SRC limit 2;
+
Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1367411&r1=1367410&r2=1367411&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Tue Jul 31 05:34:03 2012
@@ -76,6 +76,7 @@ if
in
in_file
index
+inline
instr
isnotnull
isnull
@@ -202,6 +203,7 @@ e
explode
from_unixtime
in_file
+inline
json_tuple
lcase
like
Added: hive/trunk/ql/src/test/results/clientpositive/udf_inline.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_inline.q.out?rev=1367411&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_inline.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_inline.q.out Tue Jul 31 05:34:03 2012
@@ -0,0 +1,77 @@
+PREHOOK: query: describe function inline
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function inline
+POSTHOOK: type: DESCFUNCTION
+inline( ARRAY( STRUCT()[,STRUCT()] - explodes and array and struct into a table
+PREHOOK: query: explain SELECT inline(
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ )
+) as (id, text) FROM SRC limit 2
+PREHOOK: type: QUERY
+POSTHOOK: query: explain SELECT inline(
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ )
+) as (id, text) FROM SRC limit 2
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME SRC))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION inline (TOK_FUNCTION ARRAY (TOK_FUNCTION STRUCT 1 'dude!') (TOK_FUNCTION STRUCT 2 'Wheres') (TOK_FUNCTION STRUCT 3 'my car?'))) id text)) (TOK_LIMIT 2)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: array(struct(1,'dude!'),struct(2,'Wheres'),struct(3,'my car?'))
+ type: array<struct<col1:int,col2:string>>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: inline
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 2
+
+
+PREHOOK: query: SELECT inline(
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ )
+) as (id, text) FROM SRC limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT inline(
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ )
+) as (id, text) FROM SRC limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1 dude!
+2 Wheres