You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2023/07/21 07:33:13 UTC

[hive] branch master updated: HIVE-27437: Vectorization: VectorizedOrcRecordReader does not reset VectorizedRowBatch after processing (Alagappan Maruthappan, reviewed by Denys Kuzmenko)

This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new c80f87a5347 HIVE-27437: Vectorization: VectorizedOrcRecordReader does not reset VectorizedRowBatch after processing (Alagappan Maruthappan, reviewed by Denys Kuzmenko)
c80f87a5347 is described below

commit c80f87a53475f1ccbeed8503279855c01099ee01
Author: Alagappan Maruthappan <as...@gmail.com>
AuthorDate: Fri Jul 21 00:33:05 2023 -0700

    HIVE-27437: Vectorization: VectorizedOrcRecordReader does not reset VectorizedRowBatch after processing (Alagappan Maruthappan, reviewed by Denys Kuzmenko)
    
    Closes #4415
---
 .../hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java | 15 +++++++++++++++
 .../hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java |  2 ++
 .../parquet/vector/VectorizedParquetRecordReader.java   | 11 ++++++++---
 .../hadoop/hive/ql/exec/vector/VectorizedRowBatch.java  | 17 ++++++++++++++++-
 4 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index 236bd434a25..89289b36d60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.IntStream;
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -70,6 +71,8 @@ import org.apache.hadoop.io.Writable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.util.stream.Collectors.toList;
+
 public class VectorizedBatchUtil {
   private static final Logger LOG = LoggerFactory.getLogger(VectorizedBatchUtil.class);
 
@@ -999,4 +1002,16 @@ public class VectorizedBatchUtil {
       debugDisplayOneRow(batch, index, prefix);
     }
   }
+
+  /**
+   * Reset all columns other than Partition columns
+   * @param rowBatch VectorizedRowBatch to reset
+   */
+  public static void resetNonPartitionColumns(VectorizedRowBatch rowBatch) {
+    List<Integer> columnsToReset = IntStream.concat(
+            IntStream.range(0, rowBatch.getDataColumnCount()),
+            IntStream.range(rowBatch.getDataColumnCount() + rowBatch.getPartitionColumnCount(), rowBatch.cols.length)
+    ).boxed().collect(toList());
+    rowBatch.reset(columnsToReset);
+  }
 }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
index aea4f37180b..cc57ad2782c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
@@ -122,6 +123,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
     public boolean next(NullWritable key, VectorizedRowBatch value) throws IOException {
 
       try {
+        VectorizedBatchUtil.resetNonPartitionColumns(value);
         // Check and update partition cols if necessary. Ideally, this should be done
         // in CreateValue as the partition is constant per split. But since Hive uses
         // CombineHiveRecordReader and
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
index b56990d81fe..a7ab49c43df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/VectorizedParquetRecordReader.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.llap.LlapCacheAwareFs;
 import org.apache.hadoop.hive.llap.LlapHiveUtils;
 import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
 import org.apache.hadoop.hive.ql.io.BucketIdentifier;
@@ -104,6 +105,7 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
   private List<TypeInfo> columnTypesList;
   private VectorizedRowBatchCtx rbCtx;
   private Object[] partitionValues;
+  private boolean addPartitionCols = true;
   private Path cacheFsPath;
   private static final int MAP_DEFINITION_LEVEL_MAX = 3;
 
@@ -397,14 +399,17 @@ public class VectorizedParquetRecordReader extends ParquetRecordReaderBase
   private boolean nextBatch(VectorizedRowBatch columnarBatch) throws IOException {
     currentRowNumInRowGroup += lastReturnedRowCount;
 
-    columnarBatch.reset();
+    VectorizedBatchUtil.resetNonPartitionColumns(columnarBatch);
     if (rowsReturned >= totalRowCount) {
       return false;
     }
 
     // Add partition cols if necessary (see VectorizedOrcInputFormat for details).
-    if (partitionValues != null) {
-      rbCtx.addPartitionColsToBatch(columnarBatch, partitionValues);
+    if (addPartitionCols) {
+      if (partitionValues != null) {
+        rbCtx.addPartitionColsToBatch(columnarBatch, partitionValues);
+      }
+      addPartitionCols = false;
     }
     checkEndOfRowGroup();
 
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
index f1453e80068..2ddd473293b 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.hive.ql.exec.vector;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.IntStream;
 
 import org.apache.hadoop.hive.ql.io.filter.MutableFilterContext;
 import org.apache.hadoop.io.NullWritable;
@@ -342,10 +345,22 @@ public class VectorizedRowBatch implements Writable, MutableFilterContext {
    */
   @Override
   public void reset() {
+    reset(cols);
+  }
+
+  public void reset(Collection<Integer> columnIndices) {
+    ColumnVector[] columnsToReset = columnIndices.stream()
+            .filter(index -> index < cols.length)
+            .map(x -> cols[x])
+            .toArray(ColumnVector[]::new);
+    reset(columnsToReset);
+  }
+
+  private void reset(ColumnVector[] columns) {
     selectedInUse = false;
     size = 0;
     endOfFile = false;
-    for (ColumnVector vc : cols) {
+    for (ColumnVector vc : columns) {
       if (vc != null) {
         vc.reset();
         vc.init();