You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/06/26 08:10:03 UTC
[1/2] hive git commit: HIVE-19946:
VectorizedRowBatchCtx.recordIdColumnVector cannot be shared between different
JVMs (Teddy Choi via Sergey Shelukhin, Matt McCline)
Repository: hive
Updated Branches:
refs/heads/master eaf416ea1 -> 4abc64c9c
HIVE-19946: VectorizedRowBatchCtx.recordIdColumnVector cannot be shared between different JVMs (Teddy Choi via Sergey Shelukhin, Matt McCline)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d1fb780e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d1fb780e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d1fb780e
Branch: refs/heads/master
Commit: d1fb780e5611807c20818bfe127e9b1283b6ea45
Parents: eaf416e
Author: Teddy Choi <pu...@gmail.com>
Authored: Tue Jun 26 09:26:28 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jun 26 09:26:28 2018 +0200
----------------------------------------------------------------------
.../hive/ql/exec/vector/VectorMapOperator.java | 5 ++---
.../hive/ql/exec/vector/VectorizedRowBatchCtx.java | 15 ---------------
.../ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 4 ++--
3 files changed, 4 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d1fb780e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
index 2542e03..bd70991 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java
@@ -803,10 +803,9 @@ public class VectorMapOperator extends AbstractMapOperator {
VectorizedRowBatch batch = (VectorizedRowBatch) value;
numRows += batch.size;
if (hasRowIdentifier) {
- if (batchContext.getRecordIdColumnVector() == null) {
+ final int idx = batchContext.findVirtualColumnNum(VirtualColumn.ROWID);
+ if (idx < 0) {
setRowIdentiferToNull(batch);
- } else {
- batch.cols[rowIdentifierColumnNum] = batchContext.getRecordIdColumnVector();
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d1fb780e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index ffbfb6f..c0ae7c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@ -84,13 +84,6 @@ public class VectorizedRowBatchCtx {
private int partitionColumnCount;
private int virtualColumnCount;
private VirtualColumn[] neededVirtualColumns;
- /**
- * A record ID column is a virtual column, so it should be separated from normal data column
- * processes. A recordIdColumnVector contains RecordIdentifier information in a
- * StructColumnVector. It has three LongColumnVectors as its fields; original write IDs,
- * bucket IDs, and row IDs.
- */
- private StructColumnVector recordIdColumnVector;
private String[] scratchColumnTypeNames;
private DataTypePhysicalVariation[] scratchDataTypePhysicalVariations;
@@ -217,14 +210,6 @@ public class VectorizedRowBatchCtx {
return scratchDataTypePhysicalVariations;
}
- public StructColumnVector getRecordIdColumnVector() {
- return this.recordIdColumnVector;
- }
-
- public void setRecordIdColumnVector(StructColumnVector recordIdColumnVector) {
- this.recordIdColumnVector = recordIdColumnVector;
- }
-
/**
* Initializes the VectorizedRowBatch context based on an scratch column type names and
* object inspector.
http://git-wip-us.apache.org/repos/asf/hive/blob/d1fb780e/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index e181d8c..1841cfa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -483,8 +483,8 @@ public class VectorizedOrcAcidRowBatchReader
}
}
if (rowIdProjected) {
- // TODO: could we just do: int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID); value.cols[ix] = recordIdColumnVector;
- rbCtx.setRecordIdColumnVector(recordIdColumnVector);
+ int ix = rbCtx.findVirtualColumnNum(VirtualColumn.ROWID);
+ value.cols[ix] = recordIdColumnVector;
}
progress = baseReader.getProgress();
return true;
[2/2] hive git commit: HIVE-19888: Misleading "METASTORE_FILTER_HOOK
will be ignored" warning from SessionState (Marcelo Vanzin via Zoltan
Haindrich)
Posted by kg...@apache.org.
HIVE-19888: Misleading "METASTORE_FILTER_HOOK will be ignored" warning from SessionState (Marcelo Vanzin via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4abc64c9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4abc64c9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4abc64c9
Branch: refs/heads/master
Commit: 4abc64c9c0e07e219d32c7edf09447b101076e28
Parents: d1fb780
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Tue Jun 26 09:34:04 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jun 26 09:34:04 2018 +0200
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4abc64c9/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 81864f5..6762ee6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -949,10 +949,10 @@ public class SessionState {
if (sessionConf.get(CONFIG_AUTHZ_SETTINGS_APPLIED_MARKER, "").equals(Boolean.TRUE.toString())) {
return;
}
- String metastoreHook = sessionConf.get(ConfVars.METASTORE_FILTER_HOOK.name());
+ String metastoreHook = sessionConf.getVar(ConfVars.METASTORE_FILTER_HOOK);
if (!ConfVars.METASTORE_FILTER_HOOK.getDefaultValue().equals(metastoreHook) &&
!AuthorizationMetaStoreFilterHook.class.getName().equals(metastoreHook)) {
- LOG.warn(ConfVars.METASTORE_FILTER_HOOK.name() +
+ LOG.warn(ConfVars.METASTORE_FILTER_HOOK.varname +
" will be ignored, since hive.security.authorization.manager" +
" is set to instance of HiveAuthorizerFactory.");
}