You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2016/04/13 06:20:39 UTC
hive git commit: HIVE-11484 : Fix ObjectInspector for Char and
VarChar for null and string objects (Deepak Barr, reviewed by Amareshwari)
Repository: hive
Updated Branches:
refs/heads/master 94c19741b -> ddab69c49
HIVE-11484 : Fix ObjectInspector for Char and VarChar for null and string objects (Deepak Barr, reviewed by Amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ddab69c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ddab69c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ddab69c4
Branch: refs/heads/master
Commit: ddab69c495ce6c325d7e06cfb0c704546e0d56b9
Parents: 94c1974
Author: Deepak Barr <de...@gmail.com>
Authored: Wed Apr 13 09:50:24 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Apr 13 09:50:24 2016 +0530
----------------------------------------------------------------------
.../hadoop/hive/common/type/HiveBaseChar.java | 6 ++
.../hadoop/hive/common/type/HiveVarchar.java | 4 +-
.../hive/common/type/TestHiveBaseChar.java | 2 +
.../primitive/JavaHiveCharObjectInspector.java | 15 +++-
.../JavaHiveVarcharObjectInspector.java | 15 +++-
.../PrimitiveObjectInspectorConverter.java | 8 +-
.../TestObjectInspectorConverters.java | 95 ++++++++++++++++++++
.../TestStandardObjectInspectors.java | 14 ++-
8 files changed, 148 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
index 3514726..53684e7 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveBaseChar.java
@@ -38,6 +38,9 @@ public abstract class HiveBaseChar {
}
public static String enforceMaxLength(String val, int maxLength) {
+ if (val == null) {
+ return null;
+ }
String value = val;
if (maxLength > 0) {
@@ -52,6 +55,9 @@ public abstract class HiveBaseChar {
}
public static String getPaddedValue(String val, int maxLength) {
+ if (val == null) {
+ return null;
+ }
if (maxLength < 0) {
return val;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
index 969d474..09009eb 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java
@@ -58,10 +58,10 @@ public class HiveVarchar extends HiveBaseChar
return this.getValue().compareTo(rhs.getValue());
}
- public boolean equals(HiveVarchar rhs) {
+ public boolean equals(Object rhs) {
if (rhs == this) {
return true;
}
- return this.getValue().equals(rhs.getValue());
+ return this.getValue().equals(((HiveVarchar)rhs).getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
index 2848465..98ad74a 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveBaseChar.java
@@ -80,6 +80,7 @@ public class TestHiveBaseChar extends TestCase {
assertEquals(strLen, enforcedString.codePointCount(0, enforcedString.length()));
}
}
+ assertNull(HiveBaseChar.enforceMaxLength(null, 0));
}
public void testGetPaddedValue() {
@@ -96,5 +97,6 @@ public class TestHiveBaseChar extends TestCase {
assertEquals("abc ", HiveBaseChar.getPaddedValue("abc", 10));
assertEquals("abc ", HiveBaseChar.getPaddedValue("abc ", 10));
+ assertNull(HiveBaseChar.getPaddedValue(null, 0));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
index f429709..3d89c92 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveCharObjectInspector.java
@@ -37,7 +37,12 @@ public class JavaHiveCharObjectInspector extends AbstractPrimitiveJavaObjectInsp
if (o == null) {
return null;
}
- HiveChar value = (HiveChar) o;
+ HiveChar value;
+ if (o instanceof String) {
+ value = new HiveChar((String) o, getMaxLength());
+ } else {
+ value = (HiveChar) o;
+ }
if (BaseCharUtils.doesPrimitiveMatchTypeParams(value, (CharTypeInfo) typeInfo)) {
return value;
}
@@ -49,7 +54,13 @@ public class JavaHiveCharObjectInspector extends AbstractPrimitiveJavaObjectInsp
if (o == null) {
return null;
}
- return getWritableWithParams((HiveChar) o);
+ HiveChar var;
+ if (o instanceof String) {
+ var = new HiveChar((String) o, getMaxLength());
+ } else {
+ var = (HiveChar) o;
+ }
+ return getWritableWithParams(var);
}
private HiveChar getPrimitiveWithParams(HiveChar val) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
index a8e34ff..2320d2d 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaHiveVarcharObjectInspector.java
@@ -38,7 +38,12 @@ public class JavaHiveVarcharObjectInspector extends AbstractPrimitiveJavaObjectI
if (o == null) {
return null;
}
- HiveVarchar value = (HiveVarchar) o;
+ HiveVarchar value;
+ if (o instanceof String) {
+ value= new HiveVarchar((String)o, getMaxLength());
+ } else {
+ value = (HiveVarchar)o;
+ }
if (BaseCharUtils.doesPrimitiveMatchTypeParams(value, (VarcharTypeInfo) typeInfo)) {
return value;
}
@@ -51,7 +56,13 @@ public class JavaHiveVarcharObjectInspector extends AbstractPrimitiveJavaObjectI
if (o == null) {
return null;
}
- return getWritableWithParams((HiveVarchar) o);
+ HiveVarchar var;
+ if (o instanceof String) {
+ var= new HiveVarchar((String)o, getMaxLength());
+ } else {
+ var = (HiveVarchar)o;
+ }
+ return getWritableWithParams(var);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
index 5eb41d5..e08ad43 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java
@@ -514,7 +514,7 @@ public class PrimitiveObjectInspectorConverter {
PrimitiveObjectInspector inputOI;
SettableHiveVarcharObjectInspector outputOI;
- HiveVarcharWritable hc;
+ Object hc;
public HiveVarcharConverter(PrimitiveObjectInspector inputOI,
SettableHiveVarcharObjectInspector outputOI) {
@@ -528,7 +528,7 @@ public class PrimitiveObjectInspectorConverter {
//if (typeParams == null) {
// throw new RuntimeException("varchar type used without type params");
//}
- hc = new HiveVarcharWritable();
+ hc = outputOI.create(new HiveVarchar("",-1));
}
@Override
@@ -551,13 +551,13 @@ public class PrimitiveObjectInspectorConverter {
public static class HiveCharConverter implements Converter {
PrimitiveObjectInspector inputOI;
SettableHiveCharObjectInspector outputOI;
- HiveCharWritable hc;
+ Object hc;
public HiveCharConverter(PrimitiveObjectInspector inputOI,
SettableHiveCharObjectInspector outputOI) {
this.inputOI = inputOI;
this.outputOI = outputOI;
- hc = new HiveCharWritable();
+ hc = outputOI.create(new HiveChar("",-1));
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
index dd18517..2e1bb22 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
@@ -22,10 +22,15 @@ import java.util.List;
import junit.framework.TestCase;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -116,6 +121,96 @@ public class TestObjectInspectorConverters extends TestCase {
.convert(Integer.valueOf(1)));
assertEquals("DoubleConverter", null, doubleConverter.convert(null));
+ // Char
+ Converter charConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+ PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
+ assertEquals("CharConverter", new HiveChar("TRUE", -1), charConverter
+ .convert(Boolean.valueOf(true)));
+ assertEquals("CharConverter", new HiveChar("FALSE", -1), charConverter
+ .convert(Boolean.valueOf(false)));
+
+ charConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+ PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
+ assertEquals("CharConverter", new HiveCharWritable(new HiveChar("TRUE", -1)), charConverter
+ .convert(Boolean.valueOf(true)));
+ assertEquals("CharConverter", new HiveCharWritable(new HiveChar("FALSE", -1)), charConverter
+ .convert(Boolean.valueOf(false)));
+
+ charConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
+ assertEquals("CharConverter", new HiveChar("0", -1), charConverter
+ .convert(Integer.valueOf(0)));
+ assertEquals("CharConverter", new HiveChar("1", -1), charConverter
+ .convert(Integer.valueOf(1)));
+
+ charConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
+ assertEquals("CharConverter", new HiveCharWritable(new HiveChar("0", -1)), charConverter
+ .convert(Integer.valueOf(0)));
+ assertEquals("CharConverter", new HiveCharWritable(new HiveChar("1", -1)), charConverter
+ .convert(Integer.valueOf(1)));
+
+ charConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
+ assertEquals("CharConverter", new HiveChar("hive", -1), charConverter
+ .convert(String.valueOf("hive")));
+
+ charConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
+ assertEquals("CharConverter", new HiveCharWritable(new HiveChar("hive", -1)), charConverter
+ .convert(String.valueOf("hive")));
+
+ // VarChar
+ Converter varcharConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+ PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
+ assertEquals("VarCharConverter", new HiveVarchar("TRUE", -1), varcharConverter
+ .convert(Boolean.valueOf(true)));
+ assertEquals("VarCharConverter", new HiveVarchar("FALSE", -1), varcharConverter
+ .convert(Boolean.valueOf(false)));
+
+ varcharConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector,
+ PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
+ assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("TRUE", -1)), varcharConverter
+ .convert(Boolean.valueOf(true)));
+ assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("FALSE", -1)), varcharConverter
+ .convert(Boolean.valueOf(false)));
+
+ varcharConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
+ assertEquals("VarCharConverter", new HiveVarchar("0", -1), varcharConverter
+ .convert(Integer.valueOf(0)));
+ assertEquals("VarCharConverter", new HiveVarchar("1", -1), varcharConverter
+ .convert(Integer.valueOf(1)));
+
+ varcharConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
+ assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("0", -1)), varcharConverter
+ .convert(Integer.valueOf(0)));
+ assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("1", -1)), varcharConverter
+ .convert(Integer.valueOf(1)));
+
+ varcharConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
+ assertEquals("VarCharConverter", new HiveVarchar("hive", -1), varcharConverter
+ .convert(String.valueOf("hive")));
+
+ varcharConverter = ObjectInspectorConverters.getConverter(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
+ assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("hive", -1)), varcharConverter
+ .convert(String.valueOf("hive")));
+
// Text
Converter textConverter = ObjectInspectorConverters.getConverter(
PrimitiveObjectInspectorFactory.javaIntObjectInspector,
http://git-wip-us.apache.org/repos/asf/hive/blob/ddab69c4/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
index 7d87666..29906c9 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
@@ -24,9 +24,13 @@ import java.util.List;
import junit.framework.TestCase;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -118,8 +122,14 @@ public class TestStandardObjectInspectors extends TestCase {
.getClass());
}
- assertEquals(PrimitiveObjectInspectorUtils
+ if (javaClass == HiveVarchar.class) {
+ assertEquals("varchar(65535)", oi1.getTypeName());
+ } else if (javaClass == HiveChar.class) {
+ assertEquals("char(255)", oi1.getTypeName());
+ } else {
+ assertEquals(PrimitiveObjectInspectorUtils
.getTypeNameFromPrimitiveJava(javaClass), oi1.getTypeName());
+ }
} catch (Throwable e) {
e.printStackTrace();
throw e;
@@ -143,6 +153,8 @@ public class TestStandardObjectInspectors extends TestCase {
doTestJavaPrimitiveObjectInspector(DoubleWritable.class, Double.class,
(double) 1);
doTestJavaPrimitiveObjectInspector(Text.class, String.class, "a");
+ doTestJavaPrimitiveObjectInspector(HiveVarcharWritable.class, HiveVarchar.class, "a");
+ doTestJavaPrimitiveObjectInspector(HiveCharWritable.class, HiveChar.class, "a");
doTestJavaPrimitiveObjectInspector(BytesWritable.class, byte[].class, new byte[]{'3'});
} catch (Throwable e) {