You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2017/03/22 20:55:46 UTC
hive git commit: HIVE-15789: Vectorization: limit reduce
vectorization to 32Mb chunks (Teddy Choi, via Gopal V)
Repository: hive
Updated Branches:
refs/heads/master ea3be9549 -> e434f8320
HIVE-15789: Vectorization: limit reduce vectorization to 32Mb chunks (Teddy Choi, via Gopal V)
Signed-off-by: Gopal V <go...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e434f832
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e434f832
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e434f832
Branch: refs/heads/master
Commit: e434f832015a38eecb7ac7cb5d45fbc8a2ea95ca
Parents: ea3be95
Author: Teddy Choi <tc...@hortonworks.com>
Authored: Thu Mar 23 02:24:36 2017 +0530
Committer: Gopal V <go...@apache.org>
Committed: Thu Mar 23 02:25:29 2017 +0530
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java | 6 +++++-
.../apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java | 7 ++++++-
.../apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java | 5 +++++
3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e434f832/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
index 27bed9c..7eaad18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java
@@ -95,6 +95,7 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
// number of columns pertaining to keys in a vectorized row batch
private int keysColumnOffset;
private static final int BATCH_SIZE = VectorizedRowBatch.DEFAULT_SIZE;
+ private static final int BATCH_BYTES = VectorizedRowBatch.DEFAULT_BYTES;
private StructObjectInspector keyStructInspector;
private StructObjectInspector[] valueStructInspectors;
/* this is only used in the error code path */
@@ -373,6 +374,7 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
}
int rowIdx = 0;
+ int batchBytes = 0;
try {
while (values.hasNext()) {
/* deserialize value into columns */
@@ -381,11 +383,13 @@ public class SparkReduceRecordHandler extends SparkRecordHandler {
VectorizedBatchUtil.addRowToBatchFrom(valueObj, valueStructInspectors[tag], rowIdx,
keysColumnOffset, batch, buffer);
+ batchBytes += valueWritable.getLength();
rowIdx++;
- if (rowIdx >= BATCH_SIZE) {
+ if (rowIdx >= BATCH_SIZE || batchBytes > BATCH_BYTES) {
VectorizedBatchUtil.setBatchSize(batch, rowIdx);
reducer.process(batch, tag);
rowIdx = 0;
+ batchBytes = 0;
if (isLogInfoEnabled) {
logMemoryInfo();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e434f832/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
index ad8b9e0..d9caa47 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java
@@ -103,6 +103,8 @@ public class ReduceRecordSource implements RecordSource {
// number of columns pertaining to keys in a vectorized row batch
private int firstValueColumnOffset;
+ private final int BATCH_BYTES = VectorizedRowBatch.DEFAULT_BYTES;
+
private StructObjectInspector keyStructInspector;
private StructObjectInspector valueStructInspectors;
@@ -435,6 +437,7 @@ public class ReduceRecordSource implements RecordSource {
final int maxSize = batch.getMaxSize();
Preconditions.checkState(maxSize > 0);
int rowIdx = 0;
+ int batchBytes = keyBytes.length;
try {
for (Object value : values) {
if (valueLazyBinaryDeserializeToRow != null) {
@@ -442,6 +445,7 @@ public class ReduceRecordSource implements RecordSource {
BytesWritable valueWritable = (BytesWritable) value;
byte[] valueBytes = valueWritable.getBytes();
int valueLength = valueWritable.getLength();
+ batchBytes += valueLength;
// l4j.info("ReduceRecordSource processVectorGroup valueBytes " + valueLength + " " +
// VectorizedBatchUtil.displayBytes(valueBytes, 0, valueLength));
@@ -450,7 +454,7 @@ public class ReduceRecordSource implements RecordSource {
valueLazyBinaryDeserializeToRow.deserialize(batch, rowIdx);
}
rowIdx++;
- if (rowIdx >= maxSize) {
+ if (rowIdx >= maxSize || batchBytes >= BATCH_BYTES) {
// Batch is full.
batch.size = rowIdx;
@@ -462,6 +466,7 @@ public class ReduceRecordSource implements RecordSource {
batch.cols[i].reset();
}
rowIdx = 0;
+ batchBytes = 0;
}
}
if (rowIdx > 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e434f832/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
index 278865f..5e5f13d 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -59,6 +59,11 @@ public class VectorizedRowBatch implements Writable {
*/
public static final int DEFAULT_SIZE = 1024;
+ /*
+ * This number is a safety limit for 32MB of writables.
+ */
+ public static final int DEFAULT_BYTES = 32 * 1024 * 1024;
+
/**
* Return a batch with the specified number of columns.
* This is the standard constructor -- all batches should be the same size