You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2015/12/17 01:06:02 UTC
hive git commit: HIVE-12610: Hybrid Grace Hash Join should fail task
faster if processing first batch fails,
instead of continuing processing the rest (Wei Zheng via Vikram Dixit K)
Repository: hive
Updated Branches:
refs/heads/branch-1 26728a8a3 -> 35e714304
HIVE-12610: Hybrid Grace Hash Join should fail task faster if processing first batch fails, instead of continuing processing the rest (Wei Zheng via Vikram Dixit K)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/35e71430
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/35e71430
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/35e71430
Branch: refs/heads/branch-1
Commit: 35e7143043fea446b1bb4e7b7603640d6314980c
Parents: 26728a8
Author: vikram <vi...@hortonworks.com>
Authored: Wed Dec 16 16:04:52 2015 -0800
Committer: vikram <vi...@hortonworks.com>
Committed: Wed Dec 16 16:04:52 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/MapJoinOperator.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/35e71430/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index aa51349..77dee67 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
+import com.esotericsoftware.kryo.KryoException;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
@@ -504,6 +505,11 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
if (hashPartitions[i].isHashMapOnDisk()) {
try {
continueProcess(i); // Re-process spilled data
+ } catch (KryoException ke) {
+ LOG.error("Processing the spilled data failed due to Kryo error!");
+ LOG.error("Cleaning up all spilled data!");
+ cleanupGraceHashJoin();
+ throw new HiveException(ke);
} catch (Exception e) {
throw new HiveException(e);
}
@@ -648,6 +654,19 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
}
/**
+ * Clean up data participating the join, i.e. in-mem and on-disk files for small table(s) and big table
+ */
+ private void cleanupGraceHashJoin() {
+ for (byte pos = 0; pos < mapJoinTables.length; pos++) {
+ if (pos != conf.getPosBigTable()) {
+ LOG.info("Cleaning up small table data at pos: " + pos);
+ HybridHashTableContainer container = (HybridHashTableContainer) mapJoinTables[pos];
+ container.clear();
+ }
+ }
+ }
+
+ /**
* Implements the getName function for the Node Interface.
*
* @return the name of the operator