You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by tc...@apache.org on 2018/10/31 14:39:50 UTC

hive git commit: HIVE-20751: Upgrade arrow version to 0.10.0 (Shubham Chaurasia, reviewed by Teddy Choi)

Repository: hive
Updated Branches:
  refs/heads/master fb2d27ea2 -> 4b40f7d2b


HIVE-20751: Upgrade arrow version to 0.10.0 (Shubham Chaurasia, reviewed by Teddy Choi)

Signed-off-by: Teddy Choi <tc...@hortonworks.com>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4b40f7d2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4b40f7d2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4b40f7d2

Branch: refs/heads/master
Commit: 4b40f7d2b35df26af773ec138fdf521231792cdf
Parents: fb2d27e
Author: Teddy Choi <tc...@hortonworks.com>
Authored: Wed Oct 31 23:37:41 2018 +0900
Committer: Teddy Choi <tc...@hortonworks.com>
Committed: Wed Oct 31 23:37:41 2018 +0900

----------------------------------------------------------------------
 pom.xml                                               |  2 +-
 .../hadoop/hive/llap/LlapArrowRecordWriter.java       |  4 ++--
 .../hive/ql/io/arrow/ArrowColumnarBatchSerDe.java     |  2 +-
 .../hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java |  8 ++++----
 .../apache/hadoop/hive/ql/io/arrow/Serializer.java    | 14 +++++++-------
 5 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 842a143..26b662e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,7 +123,7 @@
     <apache-directory-server.version>1.5.6</apache-directory-server.version>
     <apache-directory-clientapi.version>0.1</apache-directory-clientapi.version>
     <!-- Include arrow for LlapOutputFormatService -->
-    <arrow.version>0.8.0</arrow.version>
+    <arrow.version>0.10.0</arrow.version>
     <avatica.version>1.12.0</avatica.version>
     <avro.version>1.8.2</avro.version>
     <bonecp.version>0.8.0.RELEASE</bonecp.version>

http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
index 9ee1048..4cd8a61 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
@@ -22,9 +22,9 @@ import java.io.IOException;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 import org.apache.hadoop.hive.ql.io.arrow.ArrowWrapperWritable;
-import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.RecordWriter;
 import org.apache.hadoop.mapred.Reporter;
@@ -50,7 +50,7 @@ public class LlapArrowRecordWriter<K extends Writable, V extends Writable>
   ArrowStreamWriter arrowStreamWriter;
   WritableByteChannelAdapter out;
   BufferAllocator allocator;
-  NullableMapVector rootVector;
+  NonNullableStructVector rootVector;
 
   public LlapArrowRecordWriter(WritableByteChannelAdapter out) {
     this.out = out;

http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
index ed82d2d..0408707 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
@@ -195,7 +195,7 @@ public class ArrowColumnarBatchSerDe extends AbstractSerDe {
         for (int i = 0; i < structSize; i++) {
           structFields.add(toField(fieldNames.get(i), fieldTypeInfos.get(i)));
         }
-        return new Field(name, FieldType.nullable(MinorType.MAP.getType()), structFields);
+        return new Field(name, FieldType.nullable(MinorType.STRUCT.getType()), structFields);
       case UNION:
         final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
         final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();

http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
index 40813fa..53bee6b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.io.arrow;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.hadoop.io.WritableComparable;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -30,13 +30,13 @@ import java.io.IOException;
 public class ArrowWrapperWritable implements WritableComparable {
   private VectorSchemaRoot vectorSchemaRoot;
   private BufferAllocator allocator;
-  private NullableMapVector rootVector;
+  private NonNullableStructVector rootVector;
 
   public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot) {
     this.vectorSchemaRoot = vectorSchemaRoot;
   }
 
-  public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot, BufferAllocator allocator, NullableMapVector rootVector) {
+  public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot, BufferAllocator allocator, NonNullableStructVector rootVector) {
     this.vectorSchemaRoot = vectorSchemaRoot;
     this.allocator = allocator;
     this.rootVector = rootVector;
@@ -56,7 +56,7 @@ public class ArrowWrapperWritable implements WritableComparable {
     return allocator;
   }
 
-  public NullableMapVector getRootVector() {
+  public NonNullableStructVector getRootVector() {
     return rootVector;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
index 7dffa6b..c5079e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
@@ -36,8 +36,8 @@ import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.holders.DecimalHolder;
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.Types;
@@ -110,7 +110,7 @@ public class Serializer {
   private List<String> fieldNames;
   private int fieldSize;
 
-  private final NullableMapVector rootVector;
+  private final StructVector rootVector;
   private final DecimalHolder decimalHolder = new DecimalHolder();
 
   //Constructor for non-serde serialization
@@ -125,7 +125,7 @@ public class Serializer {
         attemptId,
         childAllocatorReservation,
         childAllocatorLimit);
-    rootVector = NullableMapVector.empty(null, allocator);
+    rootVector = StructVector.empty(null, allocator);
     //These last fields are unused in non-serde usage
     vectorizedRowBatch = null;
     vectorAssignRow = null;
@@ -151,7 +151,7 @@ public class Serializer {
     fieldNames = structTypeInfo.getAllStructFieldNames();
     fieldSize = fieldTypeInfos.size();
     // Init Arrow stuffs
-    rootVector = NullableMapVector.empty(null, allocator);
+    rootVector = StructVector.empty(null, allocator);
 
     // Init Hive stuffs
     vectorizedRowBatch = new VectorizedRowBatch(fieldSize);
@@ -292,7 +292,7 @@ public class Serializer {
         writeList((ListVector) arrowVector, (ListColumnVector) hiveVector, (ListTypeInfo) typeInfo, size, vectorizedRowBatch, isNative);
         break;
       case STRUCT:
-        writeStruct((MapVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size, vectorizedRowBatch, isNative);
+        writeStruct((NonNullableStructVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size, vectorizedRowBatch, isNative);
         break;
       case UNION:
         writeUnion(arrowVector, hiveVector, typeInfo, size, vectorizedRowBatch, isNative);
@@ -336,7 +336,7 @@ public class Serializer {
     write(arrowVector, hiveObjectVector, objectTypeInfo, size, vectorizedRowBatch, isNative);
   }
 
-  private void writeStruct(MapVector arrowVector, StructColumnVector hiveVector,
+  private void writeStruct(NonNullableStructVector arrowVector, StructColumnVector hiveVector,
       StructTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) {
     final List<String> fieldNames = typeInfo.getAllStructFieldNames();
     final List<TypeInfo> fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos();