You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/07/07 22:18:11 UTC
svn commit: r791966 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/ ql/src/test/que...
Author: namit
Date: Tue Jul 7 20:18:11 2009
New Revision: 791966
URL: http://svn.apache.org/viewvc?rev=791966&view=rev
Log:
HIVE-608, No type conversion in GenericUDF
(Zheng Shao via namit)
Removed:
hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_if_wrong_type.q
hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_if_wrong_type.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_if.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Jul 7 20:18:11 2009
@@ -312,6 +312,9 @@
HIVE-612. Problem in removing temp files in FileSinkOperator.jobClose
(Zheng Shao via namit)
+
+ HIVE-608, No type conversion in GenericUDF
+ (Zheng Shao via namit)
Release 0.3.1 - Unreleased
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Jul 7 20:18:11 2009
@@ -258,9 +258,14 @@
/**
* Find a common class that objects of both TypeInfo a and TypeInfo b can convert to.
+ * This is used for comparing objects of type a and type b.
+ *
+ * When we are comparing string and double, we will always convert both of them
+ * to double and then compare.
+ *
* @return null if no common class could be found.
*/
- public static TypeInfo getCommonClass(TypeInfo a, TypeInfo b) {
+ public static TypeInfo getCommonClassForComparison(TypeInfo a, TypeInfo b) {
// If same return one of them
if (a.equals(b)) return a;
@@ -273,6 +278,24 @@
return null;
}
+ /**
+ * Find a common class that objects of both TypeInfo a and TypeInfo b can convert to.
+ * This is used for places other than comparison.
+ *
+ * The common class of string and double is string.
+ *
+ * @return null if no common class could be found.
+ */
+ public static TypeInfo getCommonClass(TypeInfo a, TypeInfo b) {
+ Integer ai = numericTypes.get(a);
+ Integer bi = numericTypes.get(b);
+ if (ai == null || bi == null) {
+ // If either is not a numeric type, return null.
+ return null;
+ }
+ return (ai > bi) ? a : b;
+ }
+
/** Returns whether it is possible to implicitly convert an object of Class from to Class to.
*/
public static boolean implicitConvertable(TypeInfo from, TypeInfo to) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Tue Jul 7 20:18:11 2009
@@ -197,7 +197,7 @@
for (int i=1; i < newParentOps.size(); i++) {
TypeInfo a = commonType;
TypeInfo b = keyExprMap.get(new Byte((byte)i)).get(k).getTypeInfo();
- commonType = FunctionRegistry.getCommonClass(a, b);
+ commonType = FunctionRegistry.getCommonClassForComparison(a, b);
if (commonType == null) {
throw new SemanticException("Cannot do equality join on different types: " + a.getTypeName() + " and " + b.getTypeName());
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jul 7 20:18:11 2009
@@ -2810,7 +2810,7 @@
for(int i=1; i<right.length; i++) {
TypeInfo a = commonType;
TypeInfo b = keys.get(i).get(k).getTypeInfo();
- commonType = FunctionRegistry.getCommonClass(a, b);
+ commonType = FunctionRegistry.getCommonClassForComparison(a, b);
if (commonType == null) {
throw new SemanticException("Cannot do equality join on different types: " + a.getTypeName() + " and " + b.getTypeName());
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java Tue Jul 7 20:18:11 2009
@@ -41,7 +41,7 @@
public ObjectInspector initialize(ObjectInspector[] arguments)
throws UDFArgumentException {
this.argumentOIs = arguments;
- returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver();
+ returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
if (arguments.length != 3) {
throw new UDFArgumentLengthException(
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java Tue Jul 7 20:18:11 2009
@@ -18,19 +18,18 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.util.HashMap;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
@@ -46,8 +45,7 @@
*
* In many cases like CASE and IF, the GenericUDF is returning a value out
* of several possibilities. However these possibilities may not always
- * have the same ObjectInspector, although they should have the same
- * TypeInfo.
+ * have the same ObjectInspector.
*
* This class will help detect whether all possibilities have exactly the
* same ObjectInspector. If not, then we need to convert the Objects to
@@ -57,11 +55,20 @@
* can use the same ObjectInspector.
*/
public static class ReturnObjectInspectorResolver {
- boolean valueInspectorsAreTheSame;
+
+ boolean allowTypeConversion;
ObjectInspector returnObjectInspector;
- ReturnObjectInspectorResolver() {
- valueInspectorsAreTheSame = true;
+ // We create converters beforehand, so that the converters can reuse the
+ // same object for returning conversion results.
+ HashMap<ObjectInspector, ObjectInspectorUtils.Converter> converters;
+
+ public ReturnObjectInspectorResolver() {
+ this(false);
+ }
+
+ public ReturnObjectInspectorResolver(boolean allowTypeConversion) {
+ this.allowTypeConversion = allowTypeConversion;
}
/**
* Update returnObjectInspector and valueInspectorsAreTheSame based on the
@@ -70,19 +77,44 @@
*/
public boolean update(ObjectInspector oi)
throws UDFArgumentTypeException {
- if (!(oi instanceof VoidObjectInspector)) {
- if (returnObjectInspector == null) {
- returnObjectInspector = oi;
- } else if (TypeInfoUtils.getTypeInfoFromObjectInspector(oi)
- != TypeInfoUtils.getTypeInfoFromObjectInspector(returnObjectInspector)) {
- System.out.println(TypeInfoUtils.getTypeInfoFromObjectInspector(oi).getTypeName());
- System.out.println(TypeInfoUtils.getTypeInfoFromObjectInspector(returnObjectInspector).getTypeName());
- return false;
- } else {
- valueInspectorsAreTheSame = valueInspectorsAreTheSame &&
- oi == returnObjectInspector;
- }
+ if (oi instanceof VoidObjectInspector) {
+ return true;
+ }
+
+ if (returnObjectInspector == null) {
+ // The first argument, just set it.
+ returnObjectInspector = oi;
+ return true;
+ }
+
+ if (returnObjectInspector == oi) {
+ // The new ObjectInspector is the same as the old one, directly return true
+ return true;
+ }
+
+ TypeInfo oiTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(oi);
+ TypeInfo rTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(returnObjectInspector);
+ if (oiTypeInfo == rTypeInfo) {
+ // Convert everything to writable, if types of arguments are the same,
+ // but ObjectInspectors are different.
+ returnObjectInspector = ObjectInspectorUtils.getStandardObjectInspector(returnObjectInspector,
+ ObjectInspectorCopyOption.WRITABLE);
+ return true;
}
+
+ if (!allowTypeConversion) {
+ return false;
+ }
+
+ // Types are different, we need to check whether we can convert them to
+ // a common base class or not.
+ TypeInfo commonTypeInfo = FunctionRegistry.getCommonClass(oiTypeInfo, rTypeInfo);
+ if (commonTypeInfo == null) {
+ return false;
+ }
+
+ returnObjectInspector = TypeInfoUtils.getStandardObjectInspectorFromTypeInfo(commonTypeInfo);
+
return true;
}
@@ -90,10 +122,7 @@
* Returns the ObjectInspector of the return value.
*/
public ObjectInspector get() {
- return valueInspectorsAreTheSame
- ? returnObjectInspector
- : ObjectInspectorUtils.getStandardObjectInspector(returnObjectInspector,
- ObjectInspectorCopyOption.WRITABLE);
+ return returnObjectInspector;
}
/**
@@ -101,47 +130,29 @@
* different possibilities are not all the same).
*/
public Object convertIfNecessary(Object o, ObjectInspector oi) {
- if (valueInspectorsAreTheSame || oi instanceof VoidObjectInspector) {
+ if (oi == returnObjectInspector) {
return o;
} else {
- return ObjectInspectorUtils.copyToStandardObject(
- o, oi, ObjectInspectorCopyOption.WRITABLE);
+
+ if (o == null) {
+ return null;
+ }
+
+ if (converters == null) {
+ converters = new HashMap<ObjectInspector, ObjectInspectorUtils.Converter>();
+ }
+
+ Converter converter = converters.get(oi);
+ if (converter == null) {
+ converter = ObjectInspectorUtils.getConverter(oi, returnObjectInspector);
+ converters.put(oi, converter);
+ }
+
+ return converter.convert(o);
}
}
}
- /**
- * This class helps to make sure the TypeInfo of different possibilities
- * of the return values are all the same.
- */
- public static class ReturnTypeInfoResolver {
-
- TypeInfo returnTypeInfo = null;
- /**
- * Update the return TypeInfo based on the new value TypeInfo.
- * @return false if there is a type mismatch
- */
- public boolean updateReturnTypeInfo(TypeInfo newValueTypeInfo)
- throws UDFArgumentTypeException {
- if (newValueTypeInfo == TypeInfoFactory.voidTypeInfo) {
- // do nothing
- } else if (returnTypeInfo == null) {
- returnTypeInfo = newValueTypeInfo;
- } else if (returnTypeInfo != newValueTypeInfo) {
- return false;
- } else {
- // do nothing
- }
- return true;
- }
-
- public TypeInfo getReturnTypeInfo() {
- return returnTypeInfo;
- }
-
- }
-
-
}
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_if.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_if.q?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_if.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf_if.q Tue Jul 7 20:18:11 2009
@@ -16,3 +16,16 @@
IF(IF(TRUE, NULL, FALSE), 1, 2) AS COL6
FROM src LIMIT 1;
+-- Type conversions
+EXPLAIN
+SELECT IF(TRUE, CAST(128 AS SMALLINT), CAST(1 AS TINYINT)) AS COL1,
+ IF(FALSE, 1, 1.1) AS COL2,
+ IF(FALSE, 1, 'ABC') AS COL3,
+ IF(FALSE, 'ABC', 12.3) AS COL4
+FROM src LIMIT 1;
+
+SELECT IF(TRUE, CAST(128 AS SMALLINT), CAST(1 AS TINYINT)) AS COL1,
+ IF(FALSE, 1, 1.1) AS COL2,
+ IF(FALSE, 1, 'ABC') AS COL3,
+ IF(FALSE, 'ABC', 12.3) AS COL4
+FROM src LIMIT 1;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out Tue Jul 7 20:18:11 2009
@@ -53,5 +53,55 @@
IF(IF(TRUE, NULL, FALSE), 1, 2) AS COL6
FROM src LIMIT 1
Input: default/src
-Output: file:/home/zhoumin/hive-trunk/build/ql/tmp/293930288/10000
+Output: file:/data/users/zshao/tools/553-trunk-apache-hive/build/ql/tmp/1566088912/10000
1 1 1 1 NULL 2
+query: -- Type conversions
+EXPLAIN
+SELECT IF(TRUE, CAST(128 AS SMALLINT), CAST(1 AS TINYINT)) AS COL1,
+ IF(FALSE, 1, 1.1) AS COL2,
+ IF(FALSE, 1, 'ABC') AS COL3,
+ IF(FALSE, 'ABC', 12.3) AS COL4
+FROM src LIMIT 1
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION IF TRUE (TOK_FUNCTION TOK_SMALLINT 128) (TOK_FUNCTION TOK_TINYINT 1)) COL1) (TOK_SELEXPR (TOK_FUNCTION IF FALSE 1 1.1) COL2) (TOK_SELEXPR (TOK_FUNCTION IF FALSE 1 'ABC') COL3) (TOK_SELEXPR (TOK_FUNCTION IF FALSE 'ABC' 12.3) COL4)) (TOK_LIMIT 1)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ Select Operator
+ expressions:
+ expr: if(true, UDFToShort(128), UDFToByte(1))
+ type: smallint
+ expr: if(false, 1, 1.1)
+ type: double
+ expr: if(false, 1, 'ABC')
+ type: string
+ expr: if(false, 'ABC', 12.3)
+ type: string
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+
+
+query: SELECT IF(TRUE, CAST(128 AS SMALLINT), CAST(1 AS TINYINT)) AS COL1,
+ IF(FALSE, 1, 1.1) AS COL2,
+ IF(FALSE, 1, 'ABC') AS COL3,
+ IF(FALSE, 'ABC', 12.3) AS COL4
+FROM src LIMIT 1
+Input: default/src
+Output: file:/data/users/zshao/tools/553-trunk-apache-hive/build/ql/tmp/911381670/10000
+128 1.1 ABC 12.3
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Jul 7 20:18:11 2009
@@ -25,12 +25,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.serde.Constants;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
@@ -38,15 +32,12 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.TextConverter;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
/**
* ObjectInspectorFactory is the primary way to create new ObjectInspector instances.
@@ -470,5 +461,92 @@
throw new RuntimeException("Hash code on map type not supported yet.");
}
}
-
+
+ /**
+ * A converter which will convert objects with one ObjectInspector to another.
+ * We can optimize this class by reusing the same returning object, etc, in the future.
+ *
+ * Note that the outputOI has to be a standard Writable ObjectInspector for now.
+ * In the future, we can allow standard Java ObjectInspector as well.
+ */
+ public static class Converter {
+ boolean typeConversionNeeded;
+ ObjectInspector inputOI;
+ ObjectInspector outputOI;
+ TextConverter textConverter;
+
+ Converter(ObjectInspector inputOI, ObjectInspector outputOI) {
+ this.inputOI = inputOI;
+ this.outputOI = outputOI;
+
+ if (outputOI !=
+ TypeInfoUtils.getStandardObjectInspectorFromTypeInfo(
+ TypeInfoUtils.getTypeInfoFromObjectInspector(outputOI))) {
+ throw new RuntimeException("Hive internal error: ObjectInspectorUtils.Converter"
+ + " is called with non writable outputOI!");
+ }
+
+ typeConversionNeeded =
+ !TypeInfoUtils.getTypeInfoFromObjectInspector(inputOI)
+ .equals(TypeInfoUtils.getTypeInfoFromObjectInspector(outputOI));
+
+ if (outputOI instanceof StringObjectInspector) {
+ textConverter = new TextConverter();
+ }
+ }
+
+ /**
+ * Returns an object that can be inspected by outputOI.
+ * @param o an object that can be inspected by inputOI.
+ */
+ public Object convert(Object o) {
+
+ if (o == null) {
+ return null;
+ }
+
+ if (!typeConversionNeeded) {
+ return ObjectInspectorUtils.copyToStandardObject(
+ o, inputOI, ObjectInspectorCopyOption.WRITABLE);
+ }
+
+ PrimitiveObjectInspector pInputOI = (PrimitiveObjectInspector)inputOI;
+ PrimitiveObjectInspector pOutputOI = (PrimitiveObjectInspector)outputOI;
+
+ try {
+ switch (pOutputOI.getPrimitiveCategory()) {
+ case VOID:
+ case BOOLEAN:
+ case BYTE:
+ return PrimitiveObjectInspectorUtils.getByte(o, pInputOI);
+ case SHORT:
+ return PrimitiveObjectInspectorUtils.getShort(o, pInputOI);
+ case INT:
+ return PrimitiveObjectInspectorUtils.getInt(o, pInputOI);
+ case LONG:
+ return PrimitiveObjectInspectorUtils.getLong(o, pInputOI);
+ case FLOAT:
+ return PrimitiveObjectInspectorUtils.getFloat(o, pInputOI);
+ case DOUBLE:
+ return PrimitiveObjectInspectorUtils.getDouble(o, pInputOI);
+ case STRING:
+ return textConverter.convert(o, pInputOI);
+ default: throw new RuntimeException("Hive internal error: unknown category:"
+ + pOutputOI.getPrimitiveCategory());
+ }
+ } catch (NumberFormatException e) {
+ return null;
+ }
+
+ }
+ }
+
+ /**
+ * Returns a converter that converts objects from one OI to another OI.
+ * The returned (converted) object belongs to this converter, so that it can be reused
+ * across different calls.
+ */
+ public static Converter getConverter(ObjectInspector inputOI, ObjectInspector outputOI) {
+ return new Converter(inputOI, outputOI);
+ }
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java?rev=791966&r1=791965&r2=791966&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java Tue Jul 7 20:18:11 2009
@@ -25,10 +25,12 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazy.LazyInteger;
+import org.apache.hadoop.hive.serde2.lazy.LazyLong;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.io.BooleanWritable;
@@ -344,6 +346,26 @@
/**
* Get the integer number out of a primitive object.
+ * Note that NullPointerException will be thrown if o is null.
+ * Note that NumberFormatException will be thrown if o is not a valid number.
+ */
+ public static byte getByte(Object o, PrimitiveObjectInspector oi) throws NumberFormatException {
+ return (byte)getInt(o, oi);
+ }
+
+ /**
+ * Get the integer number out of a primitive object.
+ * Note that NullPointerException will be thrown if o is null.
+ * Note that NumberFormatException will be thrown if o is not a valid number.
+ */
+ public static short getShort(Object o, PrimitiveObjectInspector oi) throws NumberFormatException {
+ return (short)getInt(o, oi);
+ }
+
+ /**
+ * Get the integer number out of a primitive object.
+ * Note that NullPointerException will be thrown if o is null.
+ * Note that NumberFormatException will be thrown if o is not a valid number.
*/
public static int getInt(Object o, PrimitiveObjectInspector oi) throws NumberFormatException {
int result = 0;
@@ -399,4 +421,215 @@
}
return result;
}
+
+ /**
+ * Get the long number out of a primitive object.
+ * Note that NullPointerException will be thrown if o is null.
+ * Note that NumberFormatException will be thrown if o is not a valid number.
+ */
+ public static long getLong(Object o, PrimitiveObjectInspector oi) throws NumberFormatException {
+ long result = 0;
+ switch (oi.getPrimitiveCategory()) {
+ case VOID: {
+ result = 0;
+ break;
+ }
+ case BOOLEAN: {
+ result = (((BooleanObjectInspector)oi).get(o) ? 1 : 0);
+ break;
+ }
+ case BYTE: {
+ result = ((ByteObjectInspector)oi).get(o);
+ break;
+ }
+ case SHORT: {
+ result = ((ShortObjectInspector)oi).get(o);
+ break;
+ }
+ case INT: {
+ result = ((IntObjectInspector)oi).get(o);
+ break;
+ }
+ case LONG: {
+ result = ((LongObjectInspector)oi).get(o);
+ break;
+ }
+ case FLOAT: {
+ result = (long)((FloatObjectInspector)oi).get(o);
+ break;
+ }
+ case DOUBLE: {
+ result = (long)((DoubleObjectInspector)oi).get(o);
+ break;
+ }
+ case STRING: {
+ StringObjectInspector soi = (StringObjectInspector)oi;
+ if (soi.preferWritable()) {
+ Text t = soi.getPrimitiveWritableObject(o);
+ result = LazyLong.parseLong(t.getBytes(), 0, t.getLength());
+ } else {
+ String s = soi.getPrimitiveJavaObject(o);
+ result = Long.parseLong(s);
+ }
+ break;
+ }
+ default: {
+ // Should never happen because we checked this in SemanticAnalyzer.getXpathOrFuncExprNodeDesc
+ throw new RuntimeException("Hive 2 Internal error: index expression is not ordinal types: "
+ + oi.getTypeName());
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get the double number out of a primitive object.
+ * Note that NullPointerException will be thrown if o is null.
+ * Note that NumberFormatException will be thrown if o is not a valid number.
+ */
+ public static double getDouble(Object o, PrimitiveObjectInspector oi) throws NumberFormatException {
+ double result = 0;
+ switch (oi.getPrimitiveCategory()) {
+ case VOID: {
+ result = 0;
+ break;
+ }
+ case BOOLEAN: {
+ result = (((BooleanObjectInspector)oi).get(o) ? 1 : 0);
+ break;
+ }
+ case BYTE: {
+ result = ((ByteObjectInspector)oi).get(o);
+ break;
+ }
+ case SHORT: {
+ result = ((ShortObjectInspector)oi).get(o);
+ break;
+ }
+ case INT: {
+ result = ((IntObjectInspector)oi).get(o);
+ break;
+ }
+ case LONG: {
+ result = ((LongObjectInspector)oi).get(o);
+ break;
+ }
+ case FLOAT: {
+ result = ((FloatObjectInspector)oi).get(o);
+ break;
+ }
+ case DOUBLE: {
+ result = ((DoubleObjectInspector)oi).get(o);
+ break;
+ }
+ case STRING: {
+ StringObjectInspector soi = (StringObjectInspector)oi;
+ String s = soi.getPrimitiveJavaObject(o);
+ result = Double.parseDouble(s);
+ break;
+ }
+ default: {
+ // Should never happen because we checked this in SemanticAnalyzer.getXpathOrFuncExprNodeDesc
+ throw new RuntimeException("Hive 2 Internal error: index expression is not ordinal types: "
+ + oi.getTypeName());
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get the float number out of a primitive object.
+ * Note that NullPointerException will be thrown if o is null.
+ * Note that NumberFormatException will be thrown if o is not a valid number.
+ */
+ public static float getFloat(Object o, PrimitiveObjectInspector oi) throws NumberFormatException {
+ return (float)getDouble(o, oi);
+ }
+
+ /**
+ * A helper class to convert any primitive to Text.
+ */
+ public static class TextConverter {
+ Text t = new Text();
+ ByteStream.Output out = new ByteStream.Output();
+
+ static byte[] trueBytes = {'T', 'R', 'U', 'E'};
+ static byte[] falseBytes = {'F', 'A', 'L', 'S', 'E'};
+
+
+ public Text convert(Object o, PrimitiveObjectInspector oi) {
+ if (o == null) {
+ return null;
+ }
+
+ switch(oi.getPrimitiveCategory()) {
+ case VOID: {
+ return null;
+ }
+ case BOOLEAN: {
+ t.set(((BooleanObjectInspector)oi).get(o) ? trueBytes : falseBytes);
+ return t;
+ }
+ case BYTE: {
+ out.reset();
+ LazyInteger.writeUTF8NoException(out, ((ByteObjectInspector)oi).get(o));
+ t.set(out.getData(), 0, out.getCount());
+ return t;
+ }
+ case SHORT: {
+ out.reset();
+ LazyInteger.writeUTF8NoException(out, ((ShortObjectInspector)oi).get(o));
+ t.set(out.getData(), 0, out.getCount());
+ return t;
+ }
+ case INT: {
+ out.reset();
+ LazyInteger.writeUTF8NoException(out, ((IntObjectInspector)oi).get(o));
+ t.set(out.getData(), 0, out.getCount());
+ return t;
+ }
+ case LONG:{
+ out.reset();
+ LazyLong.writeUTF8NoException(out, ((LongObjectInspector)oi).get(o));
+ t.set(out.getData(), 0, out.getCount());
+ return t;
+ }
+ case FLOAT: {
+ t.set(String.valueOf(((FloatObjectInspector)oi).get(o)));
+ return t;
+ }
+ case DOUBLE: {
+ t.set(String.valueOf(((DoubleObjectInspector)oi).get(o)));
+ return t;
+ }
+ case STRING: {
+ t.set(((StringObjectInspector)oi).getPrimitiveJavaObject(o));
+ return t;
+ }
+ default: {
+ throw new RuntimeException("Hive 2 Internal error: type = "
+ + oi.getTypeName());
+ }
+ }
+ }
+
+ public Text evaluate(FloatWritable i) {
+ if (i == null) {
+ return null;
+ } else {
+ t.set(i.toString());
+ return t;
+ }
+ }
+
+ public Text evaluate(DoubleWritable i) {
+ if (i == null) {
+ return null;
+ } else {
+ t.set(i.toString());
+ return t;
+ }
+ }
+
+ }
}