Author: namit
Date: Thu Mar 24 14:09:49 2011
New Revision: 1084964
URL: http://svn.apache.org/viewvc?rev=1084964&view=rev
Log:
HIVE-1966 mapjoin operator should not load hashtable for each new
inputfile if the hashtable to be loaded is already there
(Yongqiang He via namit)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1084964&r1=1084963&r2=1084964&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Mar 24 14:09:49 2011
@@ -64,6 +64,7 @@ public class MapJoinOperator extends Abs
transient int metadataKeyTag;
transient int[] metadataValueTag;
transient int maxMapJoinSize;
+ transient boolean hashTblInitedOnce;
private int bigTableAlias;
public MapJoinOperator() {
@@ -103,7 +104,7 @@ public class MapJoinOperator extends Abs
rowContainerMap.put(Byte.valueOf((byte) pos), rowContainer);
}
-
+ hashTblInitedOnce = false;
}
@Override
@@ -144,10 +145,17 @@ public class MapJoinOperator extends Abs
}
private void loadHashTable() throws HiveException {
+
+ if (!this.getExecContext().getLocalWork().getInputFileChangeSensitive()) {
+ if (hashTblInitedOnce) {
+ return;
+ } else {
+ hashTblInitedOnce = true;
+ }
+ }
+
boolean localMode = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPJT).equals("local");
String baseDir = null;
- HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue> hashtable;
- Byte pos;
String currentInputFile = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME);
LOG.info("******* Load from HashTable File: input : " + currentInputFile);
@@ -181,8 +189,8 @@ public class MapJoinOperator extends Abs
}
for (Map.Entry<Byte, HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue>> entry : mapJoinTables
.entrySet()) {
- pos = entry.getKey();
- hashtable = entry.getValue();
+ Byte pos = entry.getKey();
+ HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue> hashtable = entry.getValue();
String filePath = Utilities.generatePath(baseDir, pos, currentFileName);
Path path = new Path(filePath);
LOG.info("\tLoad back 1 hashtable file from tmp file uri:" + path.toString());
@@ -293,7 +301,7 @@ public class MapJoinOperator extends Abs
public void closeOp(boolean abort) throws HiveException {
if (mapJoinTables != null) {
- for (HashMapWrapper hashTable : mapJoinTables.values()) {
+ for (HashMapWrapper<?, ?> hashTable : mapJoinTables.values()) {
hashTable.close();
}
}