You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2014/03/13 19:27:26 UTC
svn commit: r1577274 - in
/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec: ./
mr/ persistence/ tez/ vector/
Author: sershe
Date: Thu Mar 13 18:27:26 2014
New Revision: 1577274
URL: http://svn.apache.org/r1577274
Log:
HIVE-6618 : assertion when getting reference key from loader with byte-array mapjoin key (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Modified:
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java Thu Mar 13 18:27:26 2014
@@ -35,6 +35,4 @@ public interface HashTableLoader {
void load(MapJoinTableContainer[] mapJoinTables, MapJoinTableContainerSerDe[] mapJoinTableSerdes)
throws HiveException;
-
- MapJoinKey getKeyType();
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Mar 13 18:27:26 2014
@@ -170,11 +170,25 @@ public class MapJoinOperator extends Abs
protected transient final Output outputForMapJoinKey = new Output();
protected MapJoinKey computeMapJoinKey(Object row, byte alias) throws HiveException {
- MapJoinKey refKey = (key == null ? loader.getKeyType() : key);
+ MapJoinKey refKey = getRefKey(key, alias);
return MapJoinKey.readFromRow(outputForMapJoinKey,
refKey, row, joinKeys[alias], joinKeysObjectInspectors[alias], key == refKey);
}
+ protected MapJoinKey getRefKey(MapJoinKey prevKey, byte alias) {
+ if (prevKey != null) return prevKey;
+ // We assume that since we are joining on the same key, all tables would have either
+ // optimized or non-optimized key; hence, we can pass any key in any table as reference.
+ // We do it so that MJKB could determine whether it can use optimized keys.
+ for (byte pos = 0; pos < order.length; pos++) {
+ if (pos == alias) continue;
+ MapJoinKey refKey = mapJoinTables[pos].getAnyKey();
+ if (refKey != null) return refKey;
+ }
+ return null; // All join tables have 0 keys, doesn't matter what we generate.
+ }
+
+
@Override
public void processOp(Object row, int tag) throws HiveException {
try {
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java Thu Mar 13 18:27:26 2014
@@ -155,9 +155,4 @@ public class HashTableLoader implements
Arrays.fill(tables, null);
}
-
- @Override
- public MapJoinKey getKeyType() {
- return new MapJoinKeyObject(); // always use Object-array keys
- }
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java Thu Mar 13 18:27:26 2014
@@ -102,4 +102,9 @@ public class HashMapWrapper extends Abst
metaData.put(LOAD_NAME, String.valueOf(loadFactor));
return metaData;
}
+
+ @Override
+ public MapJoinKey getAnyKey() {
+ return mHash.isEmpty() ? null : mHash.keySet().iterator().next();
+ }
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java Thu Mar 13 18:27:26 2014
@@ -35,4 +35,6 @@ public interface MapJoinTableContainer {
public void clear();
+ public MapJoinKey getAnyKey();
+
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Thu Mar 13 18:27:26 2014
@@ -124,12 +124,4 @@ public class HashTableLoader implements
lastKey = new MapJoinKeyObject(); // No rows in tables, the key type doesn't matter.
}
}
-
- @Override
- public MapJoinKey getKeyType() {
- if (lastKey == null) {
- throw new AssertionError("Should be called after loading tables");
- }
- return lastKey;
- }
}
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java?rev=1577274&r1=1577273&r2=1577274&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java Thu Mar 13 18:27:26 2014
@@ -159,7 +159,7 @@ public class VectorMapJoinOperator exten
@Override
public MapJoinKey evaluate(VectorHashKeyWrapper kw) throws HiveException {
- MapJoinKey refKey = (key == null ? loader.getKeyType() : key);
+ MapJoinKey refKey = getRefKey(key, alias);
key = MapJoinKey.readFromVector(
output, refKey, kw, keyOutputWriters, keyWrapperBatch, refKey == key);
return key;