You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/04/18 04:19:41 UTC
svn commit: r1588401 - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec:
HashTableSinkOperator.java MapJoinOperator.java Operator.java
mr/MapredLocalTask.java
Author: xuefu
Date: Fri Apr 18 02:19:41 2014
New Revision: 1588401
URL: http://svn.apache.org/r1588401
Log:
HIVE-6913: Hive unable to find the hashtable file during complex multi-staged map join (Brock via Xuefu)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Fri Apr 18 02:19:41 2014
@@ -263,7 +263,9 @@ public class HashTableSinkOperator exten
@Override
public void closeOp(boolean abort) throws HiveException {
try {
- if (mapJoinTables != null) {
+ if (mapJoinTables == null) {
+ LOG.debug("mapJoinTables is null");
+ } else {
flushToFile();
}
super.closeOp(abort);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Fri Apr 18 02:19:41 2014
@@ -248,8 +248,10 @@ public class MapJoinOperator extends Abs
storage[pos] = null;
}
}
- } catch (SerDeException e) {
- throw new HiveException(e);
+ } catch (Exception e) {
+ String msg = "Unxpected exception: " + e.getMessage();
+ LOG.error(msg, e);
+ throw new HiveException(msg, e);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Fri Apr 18 02:19:41 2014
@@ -546,6 +546,7 @@ public abstract class Operator<T extends
if(parent==null){
continue;
}
+ LOG.debug("allInitializedParentsAreClosed? parent.state = " + parent.state);
if (!(parent.state == State.CLOSE || parent.state == State.UNINIT)) {
return false;
}
@@ -565,6 +566,7 @@ public abstract class Operator<T extends
// check if all parents are finished
if (!allInitializedParentsAreClosed()) {
+ LOG.debug("Not all parent operators are closed. Not closing.");
return;
}
@@ -585,6 +587,7 @@ public abstract class Operator<T extends
}
for (Operator<? extends OperatorDesc> op : childOperators) {
+ LOG.debug("Closing child = " + op);
op.close(abort);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java Fri Apr 18 02:19:41 2014
@@ -349,14 +349,15 @@ public class MapredLocalTask extends Tas
setUpFetchOpContext(fetchOp, alias, bigTableBucket);
}
+ // get the root operator
+ Operator<? extends OperatorDesc> forwardOp = work.getAliasToWork().get(alias);
if (fetchOp.isEmptyTable()) {
//generate empty hashtable for empty table
this.generateDummyHashTable(alias, bigTableBucket);
+ forwardOp.close(false);
continue;
}
- // get the root operator
- Operator<? extends OperatorDesc> forwardOp = work.getAliasToWork().get(alias);
// walk through the operator tree
while (!forwardOp.getDone()) {
InspectableObject row = fetchOp.getNextRow();
@@ -375,6 +376,9 @@ public class MapredLocalTask extends Tas
private void initializeOperators(Map<FetchOperator, JobConf> fetchOpJobConfMap)
throws HiveException {
+ for (Map.Entry<String, Operator<? extends OperatorDesc>> entry : work.getAliasToWork().entrySet()) {
+ LOG.debug("initializeOperators: " + entry.getKey() + ", children = " + entry.getValue().getChildOperators());
+ }
// this mapper operator is used to initialize all the operators
for (Map.Entry<String, FetchWork> entry : work.getAliasToFetchWork().entrySet()) {
if (entry.getValue() == null) {
@@ -419,6 +423,7 @@ public class MapredLocalTask extends Tas
private void generateDummyHashTable(String alias, String bigBucketFileName)
throws HiveException,IOException {
+ LOG.debug("generating dummy for " + alias);
// find the (byte)tag for the map join(HashTableSinkOperator)
Operator<? extends OperatorDesc> parentOp = work.getAliasToWork().get(alias);
Operator<? extends OperatorDesc> childOp = parentOp.getChildOperators().get(0);