You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2023/07/21 07:33:13 UTC
[hive] branch master updated: HIVE-27437: Vectorization: VectorizedOrcRecordReader does not reset VectorizedRowBatch after processing (Alagappan Maruthappan, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c80f87a5347 HIVE-27437: Vectorization: VectorizedOrcRecordReader does not reset VectorizedRowBatch after processing (Alagappan Maruthappan, reviewed by Denys Kuzmenko)
c80f87a5347 is described below
commit c80f87a53475f1ccbeed8503279855c01099ee01
Author: Alagappan Maruthappan <as...@gmail.com>
AuthorDate: Fri Jul 21 00:33:05 2023 -0700
HIVE-27437: Vectorization: VectorizedOrcRecordReader does not reset VectorizedRowBatch after processing (Alagappan Maruthappan, reviewed by Denys Kuzmenko)
Closes #4415
---
.../hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java | 15 +++++++++++++++
.../hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java | 2 ++
.../parquet/vector/VectorizedParquetRecordReader.java | 11 ++++++++---
.../hadoop/hive/ql/exec/vector/VectorizedRowBatch.java | 17 ++++++++++++++++-
4 files changed, 41 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index 236bd434a25..89289b36d60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.stream.IntStream;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
@@ -70,6 +71,8 @@ import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.stream.Collectors.toList;
+
public class VectorizedBatchUtil {
private static final Logger LOG = LoggerFactory.getLogger(VectorizedBatchUtil.class);
@@ -999,4 +1002,16 @@ public class VectorizedBatchUtil {
debugDisplayOneRow(batch, index, prefix);
}
}
+
+ /**
+ * Reset all columns other than Partition columns
+ * @param rowBatch VectorizedRowBatch to reset
+ */
+ public static void resetNonPartitionColumns(VectorizedRowBatch rowBatch) {
+ List<Integer> columnsToReset = IntStream.concat(
+ IntStream.range(0, rowBatch.getDataColumnCount()),
+ IntStream.range(rowBatch.getDataColumnCount() + rowBatch.getPartitionColumnCount(), rowBatch.cols.length)
+ ).boxed().collect(toList());
+ rowBatch.reset(columnsToReset);
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
index aea4f37180b..cc57ad2782c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
@@ -122,6 +123,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
public boolean next(NullWritable key, VectorizedRowBatch value) throws IOException {
try {
+ VectorizedBatchUtil.resetNonPartitionColumns(value);
// Check and update partition cols if necessary. Ideally, this should be done
// in CreateValue as the partition is constant per split. But since Hive uses
// CombineHiveRecordReader and
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
index b56990d81fe..a7ab49c43df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.llap.LlapCacheAwareFs;
import org.apache.hadoop.hive.llap.LlapHiveUtils;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
import org.apache.hadoop.hive.ql.io.BucketIdentifier;
@@ -104,6 +105,7 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
private List<TypeInfo> columnTypesList;
private VectorizedRowBatchCtx rbCtx;
private Object[] partitionValues;
+ private boolean addPartitionCols = true;
private Path cacheFsPath;
private static final int MAP_DEFINITION_LEVEL_MAX = 3;
@@ -397,14 +399,17 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
private boolean nextBatch(VectorizedRowBatch columnarBatch) throws IOException {
currentRowNumInRowGroup += lastReturnedRowCount;
- columnarBatch.reset();
+ VectorizedBatchUtil.resetNonPartitionColumns(columnarBatch);
if (rowsReturned >= totalRowCount) {
return false;
}
// Add partition cols if necessary (see VectorizedOrcInputFormat for details).
- if (partitionValues != null) {
- rbCtx.addPartitionColsToBatch(columnarBatch, partitionValues);
+ if (addPartitionCols) {
+ if (partitionValues != null) {
+ rbCtx.addPartitionColsToBatch(columnarBatch, partitionValues);
+ }
+ addPartitionCols = false;
}
checkEndOfRowGroup();
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
index f1453e80068..2ddd473293b 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.IntStream;
import org.apache.hadoop.hive.ql.io.filter.MutableFilterContext;
import org.apache.hadoop.io.NullWritable;
@@ -342,10 +345,22 @@ public class VectorizedRowBatch implements Writable, MutableFilterContext {
*/
@Override
public void reset() {
+ reset(cols);
+ }
+
+ public void reset(Collection<Integer> columnIndices) {
+ ColumnVector[] columnsToReset = columnIndices.stream()
+ .filter(index -> index < cols.length)
+ .map(x -> cols[x])
+ .toArray(ColumnVector[]::new);
+ reset(columnsToReset);
+ }
+
+ private void reset(ColumnVector[] columns) {
selectedInUse = false;
size = 0;
endOfFile = false;
- for (ColumnVector vc : cols) {
+ for (ColumnVector vc : columns) {
if (vc != null) {
vc.reset();
vc.init();