You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/01/06 17:19:22 UTC
svn commit: r896503 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/
ql/src/test/queries/clientnegative/ ql/src/test/queries/clientpositive/
ql/src/test/results/clientnegative/ q...
Author: namit
Date: Wed Jan 6 16:19:22 2010
New Revision: 896503
URL: http://svn.apache.org/viewvc?rev=896503&view=rev
Log:
HIVE-683 Add UDF field
(Larry Odrognek via namit)
Added:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java
hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_args_len.q
hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_type.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_field.q
hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_args_len.q.out
hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_type.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_field.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=896503&r1=896502&r2=896503&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Jan 6 16:19:22 2010
@@ -95,6 +95,9 @@
HIVE-1020 Create a confifuration variable to control speculative execution for reducers in hive
(namit via He Yongqiang)
+ HIVE-683 Add UDF field
+ (Larry Odrognek via namit)
+
IMPROVEMENTS
HIVE-760. Add version info to META-INF/MANIFEST.MF.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=896503&r1=896502&r2=896503&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Wed Jan 6 16:19:22 2010
@@ -105,6 +105,7 @@
registerUDF("rtrim", UDFRTrim.class, false);
registerUDF("length", UDFLength.class, false);
registerUDF("reverse", UDFReverse.class, false);
+ registerGenericUDF("field", GenericUDFField.class);
registerUDF("find_in_set", UDFFindInSet.class, false);
registerUDF("like", UDFLike.class, true);
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java Wed Jan 6 16:19:22 2010
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.exec.description;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.IntWritable;
+
+@description(
+ name="field",
+ value = "_FUNC_(str, str1, str2, ...) - returns the index of str in the str1,str2,... list or 0 if not found",
+ extended = "All primitive types are supported, arguments are compared using str.equals(x)." +
+ " If str is NULL, the return value is 0."
+ )
+public class GenericUDFField extends GenericUDF {
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length < 2) {
+ throw new UDFArgumentException("The function FIELD(str, str1, str2, ...) needs at least two arguments.");
+ }
+
+ for (int i = 0; i < arguments.length; i++) {
+ Category category = arguments[i].getCategory();
+ if (category != Category.PRIMITIVE) {
+ throw new UDFArgumentTypeException(i,
+ "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of function FIELD is expected to a "
+ + Category.PRIMITIVE.toString().toLowerCase()
+ + " type, but " + category.toString().toLowerCase() + " is found");
+ }
+ }
+
+ return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
+ }
+
+ private IntWritable r = new IntWritable();
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ if (arguments[0].get() == null) {
+ r.set(0);
+ return r;
+ }
+
+ for (int i=1; i< arguments.length; i++) {
+ if (arguments[0].get().equals(arguments[i].get())) {
+ r.set(i);
+ return r;
+ }
+ }
+
+ r.set(0);
+ return r;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ assert(children.length >= 2);
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append("field(");
+ sb.append(StringUtils.join(children, ", "));
+ sb.append(")");
+
+ return sb.toString();
+ }
+}
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_args_len.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_args_len.q?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_args_len.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_args_len.q Wed Jan 6 16:19:22 2010
@@ -0,0 +1 @@
+SELECT field(3) FROM src;
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_type.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_type.q?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_type.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_field_wrong_type.q Wed Jan 6 16:19:22 2010
@@ -0,0 +1,3 @@
+FROM src_thrift
+SELECT field(1, src_thrift.lintstring)
+WHERE src_thrift.lintstring IS NOT NULL;
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_field.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_field.q?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_field.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_field.q Wed Jan 6 16:19:22 2010
@@ -0,0 +1,24 @@
+DESCRIBE FUNCTION field;
+DESCRIBE FUNCTION EXTENDED field;
+
+SELECT
+ field("x", "a", "b", "c", "d"),
+ field(NULL, "a", "b", "c", "d"),
+ field(0, 1, 2, 3, 4)
+FROM src LIMIT 1;
+
+SELECT
+ field("a", "a", "b", "c", "d"),
+ field("b", "a", "b", "c", "d"),
+ field("c", "a", "b", "c", "d"),
+ field("d", "a", "b", "c", "d"),
+ field("d", "a", "b", NULL, "d")
+FROM src LIMIT 1;
+
+SELECT
+ field(1, 1, 2, 3, 4),
+ field(2, 1, 2, 3, 4),
+ field(3, 1, 2, 3, 4),
+ field(4, 1, 2, 3, 4),
+ field(4, 1, 2, NULL, 4)
+FROM src LIMIT 1;
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_args_len.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_args_len.q.out?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_args_len.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_args_len.q.out Wed Jan 6 16:19:22 2010
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: line 1:7 Wrong Arguments 3: The function FIELD(str, str1, str2, ...) needs at least two arguments.
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_type.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_type.q.out?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_type.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_field_wrong_type.q.out Wed Jan 6 16:19:22 2010
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: line 2:16 Argument Type Mismatch lintstring: The 2nd argument of function FIELD is expected to a primitive type, but list is found
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=896503&r1=896502&r2=896503&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Wed Jan 6 16:19:22 2010
@@ -46,6 +46,7 @@
elt
exp
explode
+field
find_in_set
float
floor
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_field.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_field.q.out?rev=896503&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_field.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_field.q.out Wed Jan 6 16:19:22 2010
@@ -0,0 +1,70 @@
+PREHOOK: query: DESCRIBE FUNCTION field
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION field
+POSTHOOK: type: DESCFUNCTION
+field(str, str1, str2, ...) - returns the index of str in the str1,str2,... list or 0 if not found
+PREHOOK: query: DESCRIBE FUNCTION EXTENDED field
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION EXTENDED field
+POSTHOOK: type: DESCFUNCTION
+field(str, str1, str2, ...) - returns the index of str in the str1,str2,... list or 0 if not found
+All primitive types are supported, arguments are compared using str.equals(x). If str is NULL, the return value is 0.
+PREHOOK: query: SELECT
+ field("x", "a", "b", "c", "d"),
+ field(NULL, "a", "b", "c", "d"),
+ field(0, 1, 2, 3, 4)
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/Users/larry/src/hive-trunk/build/ql/tmp/487027103/10000
+POSTHOOK: query: SELECT
+ field("x", "a", "b", "c", "d"),
+ field(NULL, "a", "b", "c", "d"),
+ field(0, 1, 2, 3, 4)
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/Users/larry/src/hive-trunk/build/ql/tmp/487027103/10000
+0 0 0
+PREHOOK: query: SELECT
+ field("a", "a", "b", "c", "d"),
+ field("b", "a", "b", "c", "d"),
+ field("c", "a", "b", "c", "d"),
+ field("d", "a", "b", "c", "d"),
+ field("d", "a", "b", NULL, "d")
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/Users/larry/src/hive-trunk/build/ql/tmp/1943111859/10000
+POSTHOOK: query: SELECT
+ field("a", "a", "b", "c", "d"),
+ field("b", "a", "b", "c", "d"),
+ field("c", "a", "b", "c", "d"),
+ field("d", "a", "b", "c", "d"),
+ field("d", "a", "b", NULL, "d")
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/Users/larry/src/hive-trunk/build/ql/tmp/1943111859/10000
+1 2 3 4 4
+PREHOOK: query: SELECT
+ field(1, 1, 2, 3, 4),
+ field(2, 1, 2, 3, 4),
+ field(3, 1, 2, 3, 4),
+ field(4, 1, 2, 3, 4),
+ field(4, 1, 2, NULL, 4)
+FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/Users/larry/src/hive-trunk/build/ql/tmp/533348342/10000
+POSTHOOK: query: SELECT
+ field(1, 1, 2, 3, 4),
+ field(2, 1, 2, 3, 4),
+ field(3, 1, 2, 3, 4),
+ field(4, 1, 2, 3, 4),
+ field(4, 1, 2, NULL, 4)
+FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/Users/larry/src/hive-trunk/build/ql/tmp/533348342/10000
+1 2 3 4 4