You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/27 02:05:37 UTC
[04/13] hive git commit: HIVE-19946:
VectorizedRowBatchCtx.recordIdColumnVector cannot be shared between different
JVMs (Teddy Choi via Sergey Shelukhin, Matt McCline)
HIVE-19946: VectorizedRowBatchCtx.recordIdColumnVector cannot be shared between different JVMs (Teddy Choi via Sergey Shelukhin, Matt McCline)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d1fb780e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d1fb780e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d1fb780e
Branch: refs/heads/master-txnstats
Commit: d1fb780e5611807c20818bfe127e9b1283b6ea45
Parents: eaf416e
Author: Teddy Choi <pu...@gmail.com>
Authored: Tue Jun 26 09:26:28 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jun 26 09:26:28 2018 +0200
----------------------------------------------------------------------
.../hive/ql/exec/vector/VectorMapOperator.java | 5 ++---
.../hive/ql/exec/vector/VectorizedRowBatchCtx.java | 15 ---------------
.../ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 4 ++--
3 files changed, 4 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d1fb780e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
index 2542e03..bd70991 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
@@ -803,10 +803,9 @@ public class VectorMapOperator extends AbstractMapOperator {
VectorizedRowBatch batch = (VectorizedRowBatch) value;
numRows += batch.size;
if (hasRowIdentifier) {
- if (batchContext.getRecordIdColumnVector() == null) {
+ final int idx = batchContext.findVirtualColumnNum(VirtualColumn.ROWID);
+ if (idx < 0) {
setRowIdentiferToNull(batch);
- } else {
- batch.cols[rowIdentifierColumnNum] = batchContext.getRecordIdColumnVector();
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d1fb780e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index ffbfb6f..c0ae7c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@ -84,13 +84,6 @@ public class VectorizedRowBatchCtx {
private int partitionColumnCount;
private int virtualColumnCount;
private VirtualColumn[] neededVirtualColumns;
- /**
- * A record ID column is a virtual column, so it should be separated from normal data column
- * processes. A recordIdColumnVector contains RecordIdentifier information in a
- * StructColumnVector. It has three LongColumnVectors as its fields; original write IDs,
- * bucket IDs, and row IDs.
- */
- private StructColumnVector recordIdColumnVector;
private String[] scratchColumnTypeNames;
private DataTypePhysicalVariation[] scratchDataTypePhysicalVariations;
@@ -217,14 +210,6 @@ public class VectorizedRowBatchCtx {
return scratchDataTypePhysicalVariations;
}
- public StructColumnVector getRecordIdColumnVector() {
- return this.recordIdColumnVector;
- }
-
- public void setRecordIdColumnVector(StructColumnVector recordIdColumnVector) {
- this.recordIdColumnVector = recordIdColumnVector;
- }
-
/**
* Initializes the VectorizedRowBatch context based on an scratch column type names and
* object inspector.
http://git-wip-us.apache.org/repos/asf/hive/blob/d1fb780e/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index e181d8c..1841cfa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -483,8 +483,8 @@ public class VectorizedOrcAcidRowBatchReader
}
}
if (rowIdProjected) {
- // TODO: could we just do: int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID); value.cols[ix] = recordIdColumnVector;
- rbCtx.setRecordIdColumnVector(recordIdColumnVector);
+ int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID);
+ value.cols[ix] = recordIdColumnVector;
}
progress = baseReader.getProgress();
return true;