You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ec...@apache.org on 2013/10/14 23:40:45 UTC
svn commit: r1532108 - in /hive/trunk: data/files/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: ecapriolo
Date: Mon Oct 14 21:40:44 2013
New Revision: 1532108
URL: http://svn.apache.org/r1532108
Log:
An explode function that includes the item's position in the array (Niko Stahl via egc)
Added:
hive/trunk/data/files/posexplode_data.txt (with props)
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java
hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q
hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Added: hive/trunk/data/files/posexplode_data.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/posexplode_data.txt?rev=1532108&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hive/trunk/data/files/posexplode_data.txt
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1532108&r1=1532107&r2=1532108&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Mon Oct 14 21:40:44 2013
@@ -432,6 +432,7 @@ public final class FunctionRegistry {
registerGenericUDTF("inline", GenericUDTFInline.class);
registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class);
registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class);
+ registerGenericUDTF("posexplode", GenericUDTFPosExplode.class);
registerGenericUDTF("stack", GenericUDTFStack.class);
//PTF declarations
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java?rev=1532108&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java Mon Oct 14 21:40:44 2013
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+/**
+ * PosExplode.
+ *
+ */
+@Description(
+ name = "posexplode",
+ value = "_FUNC_(a) - behaves like explode for arrays, "
+ + "but includes the position of items in the original array")
+public class GenericUDTFPosExplode extends GenericUDTF {
+ private ListObjectInspector listOI = null;
+ private final Object[] forwardObj = new Object[2];
+
+ @Override
+ public void close() throws HiveException {
+ }
+
+ @Override
+ public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException {
+
+ if (args.length != 1) {
+ throw new UDFArgumentException("posexplode() takes only one argument");
+ }
+
+ if (args[0].getCategory() != ObjectInspector.Category.LIST) {
+ throw new UDFArgumentException("posexplode() takes an array as a parameter");
+ }
+ listOI = (ListObjectInspector) args[0];
+
+ ArrayList<String> fieldNames = new ArrayList<String>();
+ ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
+ fieldNames.add("pos");
+ fieldNames.add("val");
+ fieldOIs.add(PrimitiveObjectInspectorFactory
+ .getPrimitiveJavaObjectInspector(PrimitiveCategory.INT));
+ fieldOIs.add(listOI.getListElementObjectInspector());
+ return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
+ }
+
+ @Override
+ public void process(Object[] o) throws HiveException {
+ List<?> list = listOI.getList(o[0]);
+ for (int i = 0; i < list.size(); i++) {
+ Object r = list.get(i);
+ forwardObj[0] = new Integer(i);
+ forwardObj[1] = r;
+ forward(forwardObj);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "posexplode";
+ }
+}
Added: hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q?rev=1532108&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/udtf_posexplode.q Mon Oct 14 21:40:44 2013
@@ -0,0 +1,15 @@
+CREATE TABLE employees (
+name STRING,
+salary FLOAT,
+subordinates ARRAY<STRING>,
+deductions MAP<STRING, FLOAT>,
+address STRUCT<street:STRING, city:STRING, state:STRING, zip:INT>);
+
+LOAD DATA LOCAL INPATH '../data/files/posexplode_data.txt' INTO TABLE employees;
+
+SELECT
+ name, pos, sub
+FROM
+ employees
+LATERAL VIEW
+ posexplode(subordinates) subView AS pos, sub;
Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1532108&r1=1532107&r2=1532108&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Mon Oct 14 21:40:44 2013
@@ -128,6 +128,7 @@ percentile
percentile_approx
pi
pmod
+posexplode
positive
pow
power
@@ -242,6 +243,7 @@ negative
ntile
parse_url_tuple
percentile
+posexplode
positive
regexp_replace
reverse
Added: hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out?rev=1532108&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_posexplode.q.out Mon Oct 14 21:40:44 2013
@@ -0,0 +1,42 @@
+PREHOOK: query: CREATE TABLE employees (
+name STRING,
+salary FLOAT,
+subordinates ARRAY<STRING>,
+deductions MAP<STRING, FLOAT>,
+address STRUCT<street:STRING, city:STRING, state:STRING, zip:INT>)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE employees (
+name STRING,
+salary FLOAT,
+subordinates ARRAY<STRING>,
+deductions MAP<STRING, FLOAT>,
+address STRUCT<street:STRING, city:STRING, state:STRING, zip:INT>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@employees
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/posexplode_data.txt' INTO TABLE employees
+PREHOOK: type: LOAD
+PREHOOK: Output: default@employees
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/posexplode_data.txt' INTO TABLE employees
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@employees
+PREHOOK: query: SELECT
+ name, pos, sub
+FROM
+ employees
+LATERAL VIEW
+ posexplode(subordinates) subView AS pos, sub
+PREHOOK: type: QUERY
+PREHOOK: Input: default@employees
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ name, pos, sub
+FROM
+ employees
+LATERAL VIEW
+ posexplode(subordinates) subView AS pos, sub
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@employees
+#### A masked pattern was here ####
+John Doe 0 Mary Smith
+John Doe 1 Todd Jones
+Mary Smith 0 Jeremy King