You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2020/04/14 18:06:43 UTC
[hive] branch master updated: HIVE-23168 : Implement MJ HashTable
contains key functionality (Panos G via Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6e13f8c HIVE-23168 : Implement MJ HashTable contains key functionality (Panos G via Ashutosh Chauhan)
6e13f8c is described below
commit 6e13f8c84c0d5cdaf22297f9d81c1772a413fbb4
Author: Panagiotis Garefalakis <pa...@cloudera.com>
AuthorDate: Tue Apr 14 11:05:57 2020 -0700
HIVE-23168 : Implement MJ HashTable contains key functionality (Panos G via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
.../vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java | 6 ++++++
.../exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java | 5 +++++
.../vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java | 5 +++++
.../exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java | 5 +++++
.../exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java | 5 +++++
.../ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java | 9 +++++++++
.../mapjoin/optimized/VectorMapJoinOptimizedHashTable.java | 6 ++++++
.../hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java | 2 +-
8 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
index 79b39b4..80f4546 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
@@ -53,6 +53,12 @@ public abstract class VectorMapJoinFastBytesHashTable
add(keyBytes, 0, keyLength, currentValue);
}
+ @Override
+ public boolean containsLongKey(long currentKey) {
+ // Only supported for Long-Hash implementations
+ throw new RuntimeException("Not supported yet!");
+ }
+
public abstract void add(byte[] keyBytes, int keyStart, int keyLength,
BytesWritable currentValue);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
index f9074f3..bfc829f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
@@ -155,6 +155,11 @@ public class VectorMapJoinFastLongHashMap
}
}
+ @Override
+ public boolean containsLongKey(long currentKey) {
+ return containsKey(currentKey);
+ }
+
/*
* A Unit Test convenience method for putting key and value into the hash table using the
* actual types.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java
index a9d1ed7..55f038b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java
@@ -64,6 +64,11 @@ public class VectorMapJoinFastLongHashMultiSet
}
}
+ @Override
+ public boolean containsLongKey(long currentKey) {
+ return containsKey(currentKey);
+ }
+
/*
* A Unit Test convenience method for putting the key into the hash table using the
* actual type.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java
index 10e887a..3fb9941 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java
@@ -56,6 +56,11 @@ public class VectorMapJoinFastLongHashSet
adaptPutRow(currentKey, currentValue);
}
+ @Override
+ public boolean containsLongKey(long currentKey) {
+ return containsKey(currentKey);
+ }
+
/*
* A Unit Test convenience method for putting the key into the hash table using the
* actual type.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
index b9ee7c3..d275a3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
@@ -216,6 +216,11 @@ public abstract class VectorMapJoinFastLongHashTable
metricExpands++;
}
+ protected boolean containsKey(long key) {
+ long hashCode = HashCodeUtil.calculateLongHashCode(key);
+ return findReadSlot(key, hashCode) != -1;
+ }
+
protected int findReadSlot(long key, long hashCode) {
int intHashCode = (int) hashCode;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java
index ce5c597..93e8440 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java
@@ -42,6 +42,15 @@ public interface VectorMapJoinHashTable extends MemoryEstimate {
throws SerDeException, HiveException, IOException;
/**
+ *
+ * @param currentKey
+ * The key to check for existence.
+ * @return true
+ * If HashTable contains the given key.
+ */
+ boolean containsLongKey(long currentKey);
+
+ /**
* Get hash table size
*/
int size();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java
index 45faa97..c6a9323 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java
@@ -75,6 +75,12 @@ public abstract class VectorMapJoinOptimizedHashTable
putRowInternal(currentKey, currentValue);
}
+ @Override
+ public boolean containsLongKey(long currentKey) {
+ // Method only supported by FAST HashTable implementations
+ throw new RuntimeException("Not implemented");
+ }
+
protected void putRowInternal(BytesWritable key, BytesWritable value)
throws SerDeException, HiveException, IOException {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java
index 10ed6d7..476ecd3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java
@@ -361,7 +361,7 @@ public class CheckFastRowHashMap extends CheckFastHashTable {
throw new RuntimeException("Unexpected hash table key type " + hashTableKeyType.name());
}
joinResult = longHashMap.lookup(longKey, hashMapResult);
- if (joinResult != JoinUtil.JoinResult.MATCH) {
+ if (joinResult != JoinUtil.JoinResult.MATCH || !longHashMap.containsLongKey(longKey)) {
assertTrue(false);
}
}