You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2018/08/06 21:16:32 UTC
hive git commit: HIVE-20290: Lazy initialize ArrowColumnarBatchSerDe
so it doesn't allocate buffers during GetSplits (Eric Wohlstadter,
reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 632fa4f51 -> 5935d8b63
HIVE-20290: Lazy initialize ArrowColumnarBatchSerDe so it doesn't allocate buffers during GetSplits (Eric Wohlstadter, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5935d8b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5935d8b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5935d8b6
Branch: refs/heads/master
Commit: 5935d8b63562246a7598580fd3fd73385d10ca08
Parents: 632fa4f
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Aug 6 14:15:41 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Aug 6 14:15:41 2018 -0700
----------------------------------------------------------------------
.../ql/io/arrow/ArrowColumnarBatchSerDe.java | 20 +++++++++++++++++---
1 file changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5935d8b6/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
index 470f31c..ed82d2d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/ArrowColumnarBatchSerDe.java
@@ -104,7 +104,6 @@ public class ArrowColumnarBatchSerDe extends AbstractSerDe {
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
this.conf = conf;
- rootAllocator = RootAllocatorFactory.INSTANCE.getRootAllocator(conf);
final String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
final String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
@@ -134,8 +133,6 @@ public class ArrowColumnarBatchSerDe extends AbstractSerDe {
fields.add(toField(columnNames.get(i), columnTypes.get(i)));
}
- serializer = new Serializer(this);
- deserializer = new Deserializer(this);
}
private static Field toField(String name, TypeInfo typeInfo) {
@@ -257,6 +254,15 @@ public class ArrowColumnarBatchSerDe extends AbstractSerDe {
@Override
public ArrowWrapperWritable serialize(Object obj, ObjectInspector objInspector) {
+ if(serializer == null) {
+ try {
+ rootAllocator = RootAllocatorFactory.INSTANCE.getRootAllocator(conf);
+ serializer = new Serializer(this);
+ } catch(Exception e) {
+ LOG.error("Unable to initialize serializer for ArrowColumnarBatchSerDe");
+ throw new RuntimeException(e);
+ }
+ }
return serializer.serialize(obj, objInspector);
}
@@ -267,6 +273,14 @@ public class ArrowColumnarBatchSerDe extends AbstractSerDe {
@Override
public Object deserialize(Writable writable) {
+ if(deserializer == null) {
+ try {
+ rootAllocator = RootAllocatorFactory.INSTANCE.getRootAllocator(conf);
+ deserializer = new Deserializer(this);
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
return deserializer.deserialize(writable);
}