You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by tc...@apache.org on 2018/10/31 14:39:50 UTC
hive git commit: HIVE-20751: Upgrade arrow version to 0.10.0 (Shubham
Chaurasia, reviewed by Teddy Choi)
Repository: hive
Updated Branches:
refs/heads/master fb2d27ea2 -> 4b40f7d2b
HIVE-20751: Upgrade arrow version to 0.10.0 (Shubham Chaurasia, reviewed by Teddy Choi)
Signed-off-by: Teddy Choi <tc...@hortonworks.com>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4b40f7d2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4b40f7d2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4b40f7d2
Branch: refs/heads/master
Commit: 4b40f7d2b35df26af773ec138fdf521231792cdf
Parents: fb2d27e
Author: Teddy Choi <tc...@hortonworks.com>
Authored: Wed Oct 31 23:37:41 2018 +0900
Committer: Teddy Choi <tc...@hortonworks.com>
Committed: Wed Oct 31 23:37:41 2018 +0900
----------------------------------------------------------------------
pom.xml | 2 +-
.../hadoop/hive/llap/LlapArrowRecordWriter.java | 4 ++--
.../hive/ql/io/arrow/ArrowColumnarBatchSerDe.java | 2 +-
.../hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java | 8 ++++----
.../apache/hadoop/hive/ql/io/arrow/Serializer.java | 14 +++++++-------
5 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 842a143..26b662e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,7 +123,7 @@
<apache-directory-server.version>1.5.6</apache-directory-server.version>
<apache-directory-clientapi.version>0.1</apache-directory-clientapi.version>
<!-- Include arrow for LlapOutputFormatService -->
- <arrow.version>0.8.0</arrow.version>
+ <arrow.version>0.10.0</arrow.version>
<avatica.version>1.12.0</avatica.version>
<avro.version>1.8.2</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
index 9ee1048..4cd8a61 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapArrowRecordWriter.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
import org.apache.hadoop.hive.ql.io.arrow.ArrowWrapperWritable;
-import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
@@ -50,7 +50,7 @@ public class LlapArrowRecordWriter<K extends Writable, V extends Writable>
ArrowStreamWriter arrowStreamWriter;
WritableByteChannelAdapter out;
BufferAllocator allocator;
- NullableMapVector rootVector;
+ NonNullableStructVector rootVector;
public LlapArrowRecordWriter(WritableByteChannelAdapter out) {
this.out = out;
http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
index ed82d2d..0408707 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
@@ -195,7 +195,7 @@ public class ArrowColumnarBatchSerDe extends AbstractSerDe {
for (int i = 0; i < structSize; i++) {
structFields.add(toField(fieldNames.get(i), fieldTypeInfos.get(i)));
}
- return new Field(name, FieldType.nullable(MinorType.MAP.getType()), structFields);
+ return new Field(name, FieldType.nullable(MinorType.STRUCT.getType()), structFields);
case UNION:
final UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
final List<TypeInfo> objectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
index 40813fa..53bee6b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowWrapperWritable.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.io.arrow;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
import java.io.DataInput;
import java.io.DataOutput;
@@ -30,13 +30,13 @@ import java.io.IOException;
public class ArrowWrapperWritable implements WritableComparable {
private VectorSchemaRoot vectorSchemaRoot;
private BufferAllocator allocator;
- private NullableMapVector rootVector;
+ private NonNullableStructVector rootVector;
public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot) {
this.vectorSchemaRoot = vectorSchemaRoot;
}
- public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot, BufferAllocator allocator, NullableMapVector rootVector) {
+ public ArrowWrapperWritable(VectorSchemaRoot vectorSchemaRoot, BufferAllocator allocator, NonNullableStructVector rootVector) {
this.vectorSchemaRoot = vectorSchemaRoot;
this.allocator = allocator;
this.rootVector = rootVector;
@@ -56,7 +56,7 @@ public class ArrowWrapperWritable implements WritableComparable {
return allocator;
}
- public NullableMapVector getRootVector() {
+ public NonNullableStructVector getRootVector() {
return rootVector;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4b40f7d2/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
index 7dffa6b..c5079e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
@@ -36,8 +36,8 @@ import org.apache.arrow.vector.VarBinaryVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.MapVector;
-import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.complex.NonNullableStructVector;
+import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.holders.DecimalHolder;
import org.apache.arrow.vector.types.TimeUnit;
import org.apache.arrow.vector.types.Types;
@@ -110,7 +110,7 @@ public class Serializer {
private List<String> fieldNames;
private int fieldSize;
- private final NullableMapVector rootVector;
+ private final StructVector rootVector;
private final DecimalHolder decimalHolder = new DecimalHolder();
//Constructor for non-serde serialization
@@ -125,7 +125,7 @@ public class Serializer {
attemptId,
childAllocatorReservation,
childAllocatorLimit);
- rootVector = NullableMapVector.empty(null, allocator);
+ rootVector = StructVector.empty(null, allocator);
//These last fields are unused in non-serde usage
vectorizedRowBatch = null;
vectorAssignRow = null;
@@ -151,7 +151,7 @@ public class Serializer {
fieldNames = structTypeInfo.getAllStructFieldNames();
fieldSize = fieldTypeInfos.size();
// Init Arrow stuffs
- rootVector = NullableMapVector.empty(null, allocator);
+ rootVector = StructVector.empty(null, allocator);
// Init Hive stuffs
vectorizedRowBatch = new VectorizedRowBatch(fieldSize);
@@ -292,7 +292,7 @@ public class Serializer {
writeList((ListVector) arrowVector, (ListColumnVector) hiveVector, (ListTypeInfo) typeInfo, size, vectorizedRowBatch, isNative);
break;
case STRUCT:
- writeStruct((MapVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size, vectorizedRowBatch, isNative);
+ writeStruct((NonNullableStructVector) arrowVector, (StructColumnVector) hiveVector, (StructTypeInfo) typeInfo, size, vectorizedRowBatch, isNative);
break;
case UNION:
writeUnion(arrowVector, hiveVector, typeInfo, size, vectorizedRowBatch, isNative);
@@ -336,7 +336,7 @@ public class Serializer {
write(arrowVector, hiveObjectVector, objectTypeInfo, size, vectorizedRowBatch, isNative);
}
- private void writeStruct(MapVector arrowVector, StructColumnVector hiveVector,
+ private void writeStruct(NonNullableStructVector arrowVector, StructColumnVector hiveVector,
StructTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) {
final List<String> fieldNames = typeInfo.getAllStructFieldNames();
final List<TypeInfo> fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos();