You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2015/08/28 10:28:41 UTC
[1/2] hive git commit: HIVE-10175: DynamicPartitionPruning lacks a
fast-path exit for large IN() queries (Gopal V,
reviewed by Jesus Camacho Rodriguez)
Repository: hive
Updated Branches:
refs/heads/master d147a79c1 -> ce2581680
HIVE-10175: DynamicPartitionPruning lacks a fast-path exit for large IN() queries (Gopal V, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b6d1143a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b6d1143a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b6d1143a
Branch: refs/heads/master
Commit: b6d1143aa7aaa20de035898f34df2d6b581895b6
Parents: d147a79
Author: Gopal V <go...@apache.org>
Authored: Fri Aug 28 01:22:45 2015 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri Aug 28 01:22:45 2015 -0700
----------------------------------------------------------------------
.../optimizer/DynamicPartitionPruningOptimization.java | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b6d1143a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index f475926..5ebd28a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -189,6 +189,18 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
LOG.debug("TableScan: " + ts);
}
+ if (ts == null) {
+ // could be a reduce sink
+ LOG.warn("Could not find the table scan for " + filter);
+ return null;
+ } else {
+ Table table = ts.getConf().getTableMetadata();
+ if (table != null && !table.isPartitioned()) {
+ // table is not partitioned, skip optimizer
+ return null;
+ }
+ }
+
// collect the dynamic pruning conditions
removerContext.dynLists.clear();
walkExprTree(desc.getPredicate(), removerContext);
[2/2] hive git commit: HIVE-11659: Make Vectorization use the fast
StringExpr (Gopal V, reviewed by Matt McCline)
Posted by go...@apache.org.
HIVE-11659: Make Vectorization use the fast StringExpr (Gopal V, reviewed by Matt McCline)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ce258168
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ce258168
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ce258168
Branch: refs/heads/master
Commit: ce2581680f1c109ea0a43868e0345a15b06b41c8
Parents: b6d1143
Author: Gopal V <go...@apache.org>
Authored: Fri Aug 28 01:24:32 2015 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri Aug 28 01:24:32 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java | 2 +-
.../hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java | 2 +-
.../vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java | 4 ++--
.../ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java | 4 ++--
.../exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java | 4 ++--
.../ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java | 4 ++--
6 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ce258168/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
index 626cea5..aff3551 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
@@ -143,7 +143,7 @@ public class VectorHashKeyWrapper extends KeyWrapper {
for (int i = 0; i < byteValues.length; ++i) {
// the byte comparison is potentially expensive so is better to branch on null
if (!isNull[longValues.length + doubleValues.length + i]) {
- if (0 != StringExpr.compare(
+ if (!StringExpr.equal(
byteValues[i],
byteStarts[i],
byteLengths[i],
http://git-wip-us.apache.org/repos/asf/hive/blob/ce258168/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
index a21162b..6383e8a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CuckooSetBytes.java
@@ -82,7 +82,7 @@ public class CuckooSetBytes {
}
private static boolean entryEqual(byte[][] t, int hash, byte[] b, int start, int len) {
- return t[hash] != null && StringExpr.compare(t[hash], 0, t[hash].length, b, start, len) == 0;
+ return t[hash] != null && StringExpr.equal(t[hash], 0, t[hash].length, b, start, len);
}
public void insert(byte[] x) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ce258168/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
index 87a11c0..9f2d4c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
@@ -234,8 +234,8 @@ public class VectorMapJoinInnerBigOnlyStringOperator extends VectorMapJoinInnerB
*/
if (!haveSaveKey ||
- StringExpr.compare(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
- vector[batchIndex], start[batchIndex], length[batchIndex]) != 0) {
+ StringExpr.equal(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
+ vector[batchIndex], start[batchIndex], length[batchIndex]) == false) {
// New key.
http://git-wip-us.apache.org/repos/asf/hive/blob/ce258168/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
index 9f10ff1..5a5d54f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
@@ -229,8 +229,8 @@ public class VectorMapJoinInnerStringOperator extends VectorMapJoinInnerGenerate
*/
if (!haveSaveKey ||
- StringExpr.compare(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
- vector[batchIndex], start[batchIndex], length[batchIndex]) != 0) {
+ StringExpr.equal(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
+ vector[batchIndex], start[batchIndex], length[batchIndex]) == false) {
// New key.
http://git-wip-us.apache.org/repos/asf/hive/blob/ce258168/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
index 9ff1141..e9ce739 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
@@ -230,8 +230,8 @@ public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGe
*/
if (!haveSaveKey ||
- StringExpr.compare(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
- vector[batchIndex], start[batchIndex], length[batchIndex]) != 0) {
+ StringExpr.equal(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
+ vector[batchIndex], start[batchIndex], length[batchIndex]) == false) {
// New key.
http://git-wip-us.apache.org/repos/asf/hive/blob/ce258168/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
index 49efe1a..dfdd6d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
@@ -290,8 +290,8 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
*/
if (!haveSaveKey ||
- StringExpr.compare(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
- vector[batchIndex], start[batchIndex], length[batchIndex]) != 0) {
+ StringExpr.equal(vector[saveKeyBatchIndex], start[saveKeyBatchIndex], length[saveKeyBatchIndex],
+ vector[batchIndex], start[batchIndex], length[batchIndex]) == false) {
// New key.
if (haveSaveKey) {