You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/06/04 22:47:00 UTC
[10/18] hive git commit: HIVE-10735: Cached plan race condition -
VectorMapJoinCommonOperator has no closeOp() (Matt McCline,
reviewed by Sergey Shelukhin)
HIVE-10735: Cached plan race condition - VectorMapJoinCommonOperator has no closeOp() (Matt McCline, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82beb2ba
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82beb2ba
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82beb2ba
Branch: refs/heads/llap
Commit: 82beb2ba3d0b2fa7ef690c4a51d641ced161fce2
Parents: 949ddaa
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Wed Jun 3 21:34:33 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Wed Jun 3 21:34:33 2015 -0700
----------------------------------------------------------------------
.../persistence/BytesBytesMultiHashMap.java | 48 +++++++-------------
.../persistence/MapJoinBytesTableContainer.java | 4 +-
.../ql/exec/vector/VectorFilterOperator.java | 5 +-
3 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/82beb2ba/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
index 2ba622e..3bba890 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
@@ -184,19 +184,6 @@ public final class BytesBytesMultiHashMap {
this(initialCapacity, loadFactor, wbSize, -1);
}
- public class ThreadSafeGetter {
- private WriteBuffers.Position position = new WriteBuffers.Position();
- public byte getValueResult(byte[] key, int offset, int length,
- BytesBytesMultiHashMap.Result hashMapResult) {
- return BytesBytesMultiHashMap.this.getValueResult(key, offset, length, hashMapResult, position);
- }
-
- public void populateValue(WriteBuffers.ByteSegmentRef valueRef) {
- // Convenience method, populateValue is thread-safe.
- BytesBytesMultiHashMap.this.populateValue(valueRef);
- }
- }
-
/**
* The result of looking up a key in the multi-hash map.
*
@@ -232,6 +219,14 @@ public final class BytesBytesMultiHashMap {
public Result() {
hasRows = false;
byteSegmentRef = new WriteBuffers.ByteSegmentRef();
+ readPos = new WriteBuffers.Position();
+ }
+
+ /**
+ * Return the thread-safe read position.
+ */
+ public WriteBuffers.Position getReadPos() {
+ return readPos;
}
/**
@@ -260,14 +255,11 @@ public final class BytesBytesMultiHashMap {
* Whether there are multiple values (true) or just a single value (false).
* @param offsetAfterListRecordKeyLen
* The offset of just after the key length in the list record. Or, 0 when single row.
- * @param readPos
- * Holds mutable read position for thread safety.
*/
public void set(BytesBytesMultiHashMap hashMap, long firstOffset, boolean hasList,
- long offsetAfterListRecordKeyLen, WriteBuffers.Position readPos) {
+ long offsetAfterListRecordKeyLen) {
this.hashMap = hashMap;
- this.readPos = readPos;
this.firstOffset = firstOffset;
this.hasList = hasList;
@@ -410,7 +402,6 @@ public final class BytesBytesMultiHashMap {
*/
public void forget() {
hashMap = null;
- readPos = null;
byteSegmentRef.reset(0, 0);
hasRows = false;
readIndex = 0;
@@ -481,29 +472,23 @@ public final class BytesBytesMultiHashMap {
++numValues;
}
- public ThreadSafeGetter createGetterForThread() {
- return new ThreadSafeGetter();
- }
-
- /** Not thread-safe! Use createGetterForThread. */
- public byte getValueResult(byte[] key, int offset, int length, Result hashMapResult) {
- return getValueResult(key, offset, length, hashMapResult, writeBuffers.getReadPosition());
- }
-
/**
* Finds a key. Values can be read with the supplied result object.
*
+ * Important Note: The caller is expected to pre-allocate the hashMapResult and not
+ * share it among other threads.
+ *
* @param key Key buffer.
* @param offset the offset to the key in the buffer
* @param hashMapResult The object to fill in that can read the values.
- * @param readPos Holds mutable read position for thread safety.
* @return The state byte.
*/
- private byte getValueResult(byte[] key, int offset, int length, Result hashMapResult,
- WriteBuffers.Position readPos) {
+ public byte getValueResult(byte[] key, int offset, int length, Result hashMapResult) {
hashMapResult.forget();
+ WriteBuffers.Position readPos = hashMapResult.getReadPos();
+
// First, find first record for the key.
long ref = findKeyRefToRead(key, offset, length, readPos);
if (ref == 0) {
@@ -515,8 +500,7 @@ public final class BytesBytesMultiHashMap {
// This relies on findKeyRefToRead doing key equality check and leaving read ptr where needed.
long offsetAfterListRecordKeyLen = hasList ? writeBuffers.getReadPoint(readPos) : 0;
- hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen,
- readPos);
+ hashMapResult.set(this, Ref.getOffset(ref), hasList, offsetAfterListRecordKeyLen);
return Ref.getStateByte(ref);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82beb2ba/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
index 6deaafc..34b3aa9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
@@ -500,7 +500,6 @@ public class MapJoinBytesTableContainer
private byte aliasFilter;
/** Hash table wrapper specific to the container. */
- private final BytesBytesMultiHashMap.ThreadSafeGetter threadSafeHashMapGetter;
private BytesBytesMultiHashMap.Result hashMapResult;
/**
@@ -520,14 +519,13 @@ public class MapJoinBytesTableContainer
valueStruct = null; // No rows?
}
uselessIndirection = new ByteArrayRef();
- threadSafeHashMapGetter = hashMap.createGetterForThread();
hashMapResult = new BytesBytesMultiHashMap.Result();
clearRows();
}
public JoinUtil.JoinResult setFromOutput(Output output) {
- aliasFilter = threadSafeHashMapGetter.getValueResult(
+ aliasFilter = hashMap.getValueResult(
output.getData(), 0, output.getLength(), hashMapResult);
dummyRow = null;
if (hashMapResult.hasRows()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/82beb2ba/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
index d1b8939..b40564c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFilterOperator.java
@@ -41,7 +41,7 @@ public class VectorFilterOperator extends FilterOperator {
private VectorExpression conditionEvaluator = null;
// Temporary selected vector
- private final int[] temporarySelected = new int [VectorizedRowBatch.DEFAULT_SIZE];
+ private transient int[] temporarySelected;
// filterMode is 1 if condition is always true, -1 if always false
// and 0 if condition needs to be computed.
@@ -77,6 +77,9 @@ public class VectorFilterOperator extends FilterOperator {
filterMode = -1;
}
}
+
+ temporarySelected = new int [VectorizedRowBatch.DEFAULT_SIZE];
+
return result;
}