You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2015/08/21 20:17:57 UTC
hive git commit: HIVE-11513: AvroLazyObjectInspector could handle
empty data better (Swarnim Kulkarni, reviewed by Xuefu Zhang)
Repository: hive
Updated Branches:
refs/heads/master a16267491 -> b4a6ac921
HIVE-11513: AvroLazyObjectInspector could handle empty data better (Swarnim Kulkarni, reviewed by Xuefu Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4a6ac92
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4a6ac92
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4a6ac92
Branch: refs/heads/master
Commit: b4a6ac921e0a283107d9e852116d76ba097ccee5
Parents: a162674
Author: Swarnim Kulkarni <ku...@gmail.com>
Authored: Fri Aug 21 11:17:46 2015 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Fri Aug 21 11:17:46 2015 -0700
----------------------------------------------------------------------
.../serde2/avro/AvroLazyObjectInspector.java | 19 +++----
.../avro/TestAvroLazyObjectInspector.java | 59 ++++++++++++++++++++
2 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4a6ac92/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
index 9fc9873..756f566 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java
@@ -138,7 +138,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
if (rowField instanceof LazyStruct) {
- if (LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled() && rowField != null) {
LOG.debug("Deserializing struct [" + rowField.getClass() + "]");
}
@@ -166,7 +166,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
} else {
if (LOG.isDebugEnabled()) {
- LOG.debug("Returning [" + rowField.toString() + "] for field [" + f.getFieldName() + "]");
+ LOG.debug("Returning [" + rowField + "] for field [" + f.getFieldName() + "]");
}
// Just return the object. We need no further operation on it
@@ -223,7 +223,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
byte[] data = ((LazyStruct) struct).getBytes();
AvroDeserializer deserializer = new AvroDeserializer();
- if (data == null) {
+ if (data == null || data.length == 0) {
return null;
}
@@ -239,6 +239,12 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable();
if (readerSchema == null) {
+ offset = schemaRetriever.getOffset();
+
+ if (data.length < offset) {
+ throw new IllegalArgumentException("Data size cannot be less than [" + offset
+ + "]. Found [" + data.length + "]");
+ }
rs = schemaRetriever.retrieveReaderSchema(data);
@@ -257,13 +263,6 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector {
}
// adjust the data bytes according to any possible offset that was provided
- offset = schemaRetriever.getOffset();
-
- if (data.length < offset) {
- throw new IllegalArgumentException("Data size cannot be less than [" + offset
- + "]. Found [" + data.length + "]");
- }
-
if (LOG.isDebugEnabled()) {
LOG.debug("Retrieved writer Schema: " + ws.toString());
LOG.debug("Retrieved reader Schema: " + rs.toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/b4a6ac92/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java
new file mode 100644
index 0000000..2b7d513
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java
@@ -0,0 +1,59 @@
+package org.apache.hadoop.hive.serde2.avro;
+
+import static org.junit.Assert.assertNull;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class TestAvroLazyObjectInspector {
+
+ @Test
+ public void testEmptyData(){
+ List<String> fieldNames = new ArrayList<String>();
+ fieldNames.add("myField");
+
+ List<ObjectInspector> ois = new ArrayList<ObjectInspector>();
+ ois.add(LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, new Byte((byte) 0)));
+
+ AvroLazyObjectInspector aloi = new AvroLazyObjectInspector(fieldNames, ois, null, (byte)0, new Text(), false, false, (byte)0);
+ LazyStruct lazyStruct = new LazyStruct(LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(fieldNames, ois, (byte)0, new Text(), false, false, (byte)0));
+
+ ByteArrayRef byteArrayRef = new ByteArrayRef();
+ byteArrayRef.setData(new byte[0]); // set data to empty explicitly
+ lazyStruct.init(byteArrayRef, 0, 0);
+
+ assertNull(aloi.getStructFieldData(lazyStruct, new TestStructField()));
+ }
+
+ class TestStructField implements StructField {
+
+ @Override
+ public String getFieldName() {
+ return "testfield";
+ }
+
+ @Override
+ public ObjectInspector getFieldObjectInspector() {
+ return LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, new Byte((byte) 0));
+ }
+
+ @Override
+ public int getFieldID() {
+ return 0;
+ }
+
+ @Override
+ public String getFieldComment() {
+ return null;
+ }
+ }
+}