You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/11/10 18:36:37 UTC
svn commit: r1033605 - in /hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: namit
Date: Wed Nov 10 17:36:37 2010
New Revision: 1033605
URL: http://svn.apache.org/viewvc?rev=1033605&view=rev
Log:
HIVE-1779 Add udf str_to_map
(Siying Dong via via namit)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
hive/trunk/ql/src/test/queries/clientpositive/str_to_map.q
hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out
Modified:
hive/trunk/CHANGES.txt
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
Modified: hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1033605&r1=1033604&r2=1033605&view=diff
==============================================================================
--- hive/trunk/CHANGES.txt (original)
+++ hive/trunk/CHANGES.txt Wed Nov 10 17:36:37 2010
@@ -104,6 +104,9 @@ Trunk - Unreleased
HIVE-474 Support for distinct selection on two or more columns
(Amareshwari Sriramadasu via namit)
+ HIVE-1779 Add udf str_to_map
+ (Siying Dong via via namit)
+
IMPROVEMENTS
HIVE-1394. Do not update transient_lastDdlTime if the partition is modified by a housekeeping
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1033605&r1=1033604&r2=1033605&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Wed Nov 10 17:36:37 2010
@@ -174,6 +174,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSentences;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSplit;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStringToMap;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnion;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
@@ -275,6 +276,7 @@ public final class FunctionRegistry {
registerUDF("regexp_extract", UDFRegExpExtract.class, false);
registerUDF("parse_url", UDFParseUrl.class, false);
registerGenericUDF("split", GenericUDFSplit.class);
+ registerGenericUDF("str_to_map", GenericUDFStringToMap.class);
registerUDF("positive", UDFOPPositive.class, true, "+");
registerUDF("negative", UDFOPNegative.class, true, "-");
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java?rev=1033605&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java Wed Nov 10 17:36:37 2010
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.util.HashMap;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+
+/**
+ * GenericUDFStringToMap.
+ *
+ */
+@Description(name = "str_to_map", value = "_FUNC_(text, delimiter1, delimiter2) - "
+ + "Creates a map by parsing text ")
+public class GenericUDFStringToMap extends GenericUDF {
+ HashMap<Object, Object> ret = new HashMap<Object, Object>();
+ StringObjectInspector soi_text, soi_de1 = null, soi_de2 = null;
+ final static String default_de1 = ",";
+ final static String default_de2 = ":";
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+
+ if (!TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[0]).equals(
+ TypeInfoFactory.stringTypeInfo)
+ || (arguments.length > 1 &&
+ !TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[1]).equals(
+ TypeInfoFactory.stringTypeInfo))
+ || (arguments.length > 2 &&
+ !TypeInfoUtils.getTypeInfoFromObjectInspector(arguments[2]).equals(
+ TypeInfoFactory.stringTypeInfo))) {
+ throw new UDFArgumentException("All argument should be string");
+ }
+
+ soi_text = (StringObjectInspector) arguments[0];
+ if (arguments.length > 1) {
+ soi_de1 = (StringObjectInspector) arguments[1];
+ }
+ if (arguments.length > 2) {
+ soi_de2 = (StringObjectInspector) arguments[2];
+ }
+
+ return ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ ret.clear();
+ String text = soi_text.getPrimitiveJavaObject(arguments[0].get());
+ String delimiter1 = (soi_de1 == null) ?
+ default_de1 : soi_de1.getPrimitiveJavaObject(arguments[1].get());
+ String delimiter2 = (soi_de2 == null) ?
+ default_de2 : soi_de2.getPrimitiveJavaObject(arguments[2].get());
+
+ String[] keyValuePairs = text.split(delimiter1);
+
+ for (String keyValuePair : keyValuePairs) {
+ String[] keyValue = keyValuePair.split(delimiter2, 2);
+ if (keyValue.length < 2) {
+ ret.put(keyValuePair, null);
+ } else {
+ ret.put(keyValue[0], keyValue[1]);
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("str_to_map(");
+ assert (children.length <= 3);
+ boolean firstChild = true;
+ for (String child : children) {
+ if (firstChild) {
+ firstChild = false;
+ } else {
+ sb.append(",");
+ }
+ sb.append(child);
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+}
Added: hive/trunk/ql/src/test/queries/clientpositive/str_to_map.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/str_to_map.q?rev=1033605&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/str_to_map.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/str_to_map.q Wed Nov 10 17:36:37 2010
@@ -0,0 +1,26 @@
+desc function str_to_map;
+desc function extended str_to_map;
+
+explain select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3;
+select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3;
+
+explain select str_to_map('a:1,b:2,c:3') from src limit 3;
+select str_to_map('a:1,b:2,c:3') from src limit 3;
+
+explain select str_to_map('a:1,b:2,c:3',',',':') from src limit 3;
+select str_to_map('a:1,b:2,c:3',',',':') from src limit 3;
+
+explain select str_to_map(t.ss,',',':')['a']
+from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
+limit 3;
+select str_to_map(t.ss,',',':')['a']
+from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
+limit 3;
+
+
+drop table tbl_s2m;
+create table tbl_s2m as select 'ABC=CC_333=444' as t from src limit 3;
+
+select str_to_map(t,'_','=')['333'] from tbl_s2m;
+
+drop table tbl_s2m;
Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1033605&r1=1033604&r2=1033605&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Wed Nov 10 17:36:37 2010
@@ -129,6 +129,7 @@ std
stddev
stddev_pop
stddev_samp
+str_to_map
string
struct
substr
Added: hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out?rev=1033605&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out Wed Nov 10 17:36:37 2010
@@ -0,0 +1,246 @@
+PREHOOK: query: desc function str_to_map
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: desc function str_to_map
+POSTHOOK: type: DESCFUNCTION
+str_to_map(text, delimiter1, delimiter2) - Creates a map by parsing text
+PREHOOK: query: desc function extended str_to_map
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: desc function extended str_to_map
+POSTHOOK: type: DESCFUNCTION
+str_to_map(text, delimiter1, delimiter2) - Creates a map by parsing text
+PREHOOK: query: explain select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR ([ (TOK_FUNCTION str_to_map 'a=1,b=2,c=3' ',' '=') 'a'))) (TOK_LIMIT 3)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: str_to_map('a=1,b=2,c=3',',','=')['a']
+ type: string
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 3
+
+
+PREHOOK: query: select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-28_591_6260158416663883074/-mr-10000
+POSTHOOK: query: select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-28_591_6260158416663883074/-mr-10000
+1
+1
+1
+PREHOOK: query: explain select str_to_map('a:1,b:2,c:3') from src limit 3
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select str_to_map('a:1,b:2,c:3') from src limit 3
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION str_to_map 'a:1,b:2,c:3'))) (TOK_LIMIT 3)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: str_to_map('a:1,b:2,c:3')
+ type: map<string,string>
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 3
+
+
+PREHOOK: query: select str_to_map('a:1,b:2,c:3') from src limit 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-32_534_2589249075999917145/-mr-10000
+POSTHOOK: query: select str_to_map('a:1,b:2,c:3') from src limit 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-32_534_2589249075999917145/-mr-10000
+{"b":"2","c":"3","a":"1"}
+{"b":"2","c":"3","a":"1"}
+{"b":"2","c":"3","a":"1"}
+PREHOOK: query: explain select str_to_map('a:1,b:2,c:3',',',':') from src limit 3
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select str_to_map('a:1,b:2,c:3',',',':') from src limit 3
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION str_to_map 'a:1,b:2,c:3' ',' ':'))) (TOK_LIMIT 3)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: str_to_map('a:1,b:2,c:3',',',':')
+ type: map<string,string>
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 3
+
+
+PREHOOK: query: select str_to_map('a:1,b:2,c:3',',',':') from src limit 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-36_278_460631588022401435/-mr-10000
+POSTHOOK: query: select str_to_map('a:1,b:2,c:3',',',':') from src limit 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-36_278_460631588022401435/-mr-10000
+{"b":"2","c":"3","a":"1"}
+{"b":"2","c":"3","a":"1"}
+{"b":"2","c":"3","a":"1"}
+PREHOOK: query: explain select str_to_map(t.ss,',',':')['a']
+from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
+limit 3
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select str_to_map(t.ss,',',':')['a']
+from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
+limit 3
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST 'a:1,b:2,c:3') TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER (TOK_ALIASLIST ss)))))) t)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR ([ (TOK_FUNCTION str_to_map (. (TOK_TABLE_OR_COL t) ss) ',' ':') 'a'))) (TOK_LIMIT 3)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ t:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: 'a:1,b:2,c:3'
+ type: string
+ outputColumnNames: _col0
+ Transform Operator
+ command: cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: str_to_map(_col0,',',':')['a']
+ type: string
+ outputColumnNames: _col0
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 3
+
+
+PREHOOK: query: select str_to_map(t.ss,',',':')['a']
+from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
+limit 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-40_008_1745894324808734321/-mr-10000
+POSTHOOK: query: select str_to_map(t.ss,',',':')['a']
+from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
+limit 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-40_008_1745894324808734321/-mr-10000
+1
+1
+1
+PREHOOK: query: drop table tbl_s2m
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tbl_s2m
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tbl_s2m as select 'ABC=CC_333=444' as t from src limit 3
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: default@src
+POSTHOOK: query: create table tbl_s2m as select 'ABC=CC_333=444' as t from src limit 3
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tbl_s2m
+PREHOOK: query: select str_to_map(t,'_','=')['333'] from tbl_s2m
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_s2m
+PREHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-48_099_3563726538760875441/-mr-10000
+POSTHOOK: query: select str_to_map(t,'_','=')['333'] from tbl_s2m
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_s2m
+POSTHOOK: Output: file:/tmp/njain/hive_2010-11-10_09-32-48_099_3563726538760875441/-mr-10000
+444
+444
+444
+PREHOOK: query: drop table tbl_s2m
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tbl_s2m
+PREHOOK: Output: default@tbl_s2m
+POSTHOOK: query: drop table tbl_s2m
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tbl_s2m
+POSTHOOK: Output: default@tbl_s2m