You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2017/03/22 20:55:46 UTC

hive git commit: HIVE-15789: Vectorization: limit reduce vectorization to 32Mb chunks (Teddy Choi, via Gopal V)

Repository: hive
Updated Branches:
  refs/heads/master ea3be9549 -> e434f8320


HIVE-15789: Vectorization: limit reduce vectorization to 32Mb chunks (Teddy Choi, via Gopal V)

Signed-off-by: Gopal V <go...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e434f832
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e434f832
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e434f832

Branch: refs/heads/master
Commit: e434f832015a38eecb7ac7cb5d45fbc8a2ea95ca
Parents: ea3be95
Author: Teddy Choi <tc...@hortonworks.com>
Authored: Thu Mar 23 02:24:36 2017 +0530
Committer: Gopal V <go...@apache.org>
Committed: Thu Mar 23 02:25:29 2017 +0530

----------------------------------------------------------------------
 .../hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java   | 6 +++++-
 .../apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java    | 7 ++++++-
 .../apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java | 5 +++++
 3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e434f832/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
index 27bed9c..7eaad18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
@@ -95,6 +95,7 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
   // number of columns pertaining to keys in a vectorized row batch
   private int keysColumnOffset;
   private static final int BATCH_SIZE = VectorizedRowBatch.DEFAULT_SIZE;
+  private static final int BATCH_BYTES = VectorizedRowBatch.DEFAULT_BYTES;
   private StructObjectInspector keyStructInspector;
   private StructObjectInspector[] valueStructInspectors;
   /* this is only used in the error code path */
@@ -373,6 +374,7 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
     }
 
     int rowIdx = 0;
+    int batchBytes = 0;
     try {
       while (values.hasNext()) {
         /* deserialize value into columns */
@@ -381,11 +383,13 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
 
         VectorizedBatchUtil.addRowToBatchFrom(valueObj, valueStructInspectors[tag], rowIdx,
             keysColumnOffset, batch, buffer);
+        batchBytes += valueWritable.getLength();
         rowIdx++;
-        if (rowIdx >= BATCH_SIZE) {
+        if (rowIdx >= BATCH_SIZE || batchBytes > BATCH_BYTES) {
           VectorizedBatchUtil.setBatchSize(batch, rowIdx);
           reducer.process(batch, tag);
           rowIdx = 0;
+          batchBytes = 0;
           if (isLogInfoEnabled) {
             logMemoryInfo();
           }

http://git-wip-us.apache.org/repos/asf/hive/blob/e434f832/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
index ad8b9e0..d9caa47 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
@@ -103,6 +103,8 @@ public class ReduceRecordSource implements RecordSource {
   // number of columns pertaining to keys in a vectorized row batch
   private int firstValueColumnOffset;
 
+  private final int BATCH_BYTES = VectorizedRowBatch.DEFAULT_BYTES;
+
   private StructObjectInspector keyStructInspector;
   private StructObjectInspector valueStructInspectors;
 
@@ -435,6 +437,7 @@ public class ReduceRecordSource implements RecordSource {
     final int maxSize = batch.getMaxSize();
     Preconditions.checkState(maxSize > 0);
     int rowIdx = 0;
+    int batchBytes = keyBytes.length;
     try {
       for (Object value : values) {
         if (valueLazyBinaryDeserializeToRow != null) {
@@ -442,6 +445,7 @@ public class ReduceRecordSource implements RecordSource {
           BytesWritable valueWritable = (BytesWritable) value;
           byte[] valueBytes = valueWritable.getBytes();
           int valueLength = valueWritable.getLength();
+          batchBytes += valueLength;
 
           // l4j.info("ReduceRecordSource processVectorGroup valueBytes " + valueLength + " " +
           //     VectorizedBatchUtil.displayBytes(valueBytes, 0, valueLength));
@@ -450,7 +454,7 @@ public class ReduceRecordSource implements RecordSource {
           valueLazyBinaryDeserializeToRow.deserialize(batch, rowIdx);
         }
         rowIdx++;
-        if (rowIdx >= maxSize) {
+        if (rowIdx >= maxSize || batchBytes >= BATCH_BYTES) {
 
           // Batch is full.
           batch.size = rowIdx;
@@ -462,6 +466,7 @@ public class ReduceRecordSource implements RecordSource {
             batch.cols[i].reset();
           }
           rowIdx = 0;
+          batchBytes = 0;
         }
       }
       if (rowIdx > 0) {

http://git-wip-us.apache.org/repos/asf/hive/blob/e434f832/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
index 278865f..5e5f13d 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -59,6 +59,11 @@ public class VectorizedRowBatch implements Writable {
    */
   public static final int DEFAULT_SIZE = 1024;
 
+  /*
+   * This number is a safety limit for 32MB of writables.
+   */
+  public static final int DEFAULT_BYTES = 32 * 1024 * 1024;
+
   /**
    * Return a batch with the specified number of columns.
    * This is the standard constructor -- all batches should be the same size