You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/09/25 19:32:11 UTC
svn commit: r1390010 [1/2] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/ java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/
java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/
test/queries/c...
Author: namit
Date: Tue Sep 25 17:32:09 2012
New Revision: 1390010
URL: http://svn.apache.org/viewvc?rev=1390010&view=rev
Log:
HIVE-3411 Filter predicates on outer join overlapped on single alias is not handled properly
(Navis via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/join_filters_overlap.q
hive/trunk/ql/src/test/results/clientpositive/join_filters_overlap.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/union22.q.out
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Tue Sep 25 17:32:09 2012
@@ -247,6 +247,9 @@ public enum ErrorMsg {
"Fix the metadata or don't use bucketed mapjoin, by setting " +
"hive.enforce.bucketmapjoin to false."),
+ JOINNODE_OUTERJOIN_MORETHAN_8(10142, "Single join node containing outer join(s) " +
+ "cannot have more than 8 aliases"),
+
CREATE_SKEWED_TABLE_NO_COLUMN_NAME(10200, "No skewed column name."),
CREATE_SKEWED_TABLE_NO_COLUMN_VALUE(10201, "No skewed values."),
CREATE_SKEWED_TABLE_DUPLICATE_COLUMN_NAMES(10202,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java Tue Sep 25 17:32:09 2012
@@ -96,9 +96,10 @@ public abstract class AbstractMapJoinOpe
posBigTable = conf.getPosBigTable();
emptyList = new RowContainer<ArrayList<Object>>(1, hconf);
+
RowContainer bigPosRC = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors.get((byte) posBigTable),
- order[posBigTable], joinCacheSize,spillTableDesc, conf,noOuterJoin);
+ order[posBigTable], joinCacheSize,spillTableDesc, conf, !hasFilter(posBigTable));
storage.put((byte) posBigTable, bigPosRC);
mapJoinRowsKey = HiveConf.getIntVar(hconf,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Tue Sep 25 17:32:09 2012
@@ -36,13 +36,12 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.io.BooleanWritable;
/**
* Join operator implementation.
@@ -98,6 +97,8 @@ public abstract class CommonJoinOperator
*/
protected transient Map<Byte, List<ExprNodeEvaluator>> joinFilters;
+ protected transient int[][] filterMap;
+
/**
* The ObjectInspectors for the join inputs.
*/
@@ -260,6 +261,8 @@ public abstract class CommonJoinOperator
joinValuesStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
joinValuesObjectInspectors,NOTSKIPBIGTABLE);
+ filterMap = conf.getFilterMap();
+
if (noOuterJoin) {
rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors;
} else {
@@ -270,7 +273,7 @@ public abstract class CommonJoinOperator
rcOIs.addAll(joinValuesObjectInspectors.get(alias));
// for each alias, add object inspector for boolean as the last element
rcOIs.add(
- PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
+ PrimitiveObjectInspectorFactory.writableByteObjectInspector);
rowContainerObjectInspectors.put(alias, rcOIs);
}
rowContainerStandardObjectInspectors =
@@ -304,13 +307,13 @@ public abstract class CommonJoinOperator
// add whether the row is filtered or not
// this value does not matter for the dummyObj
// because the join values are already null
- nr.add(new BooleanWritable(false));
+ nr.add(new ByteWritable());
}
dummyObj[pos] = nr;
// there should be only 1 dummy object in the RowContainer
RowContainer<ArrayList<Object>> values = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors.get((byte)pos),
- alias, 1, spillTableDesc, conf, noOuterJoin);
+ alias, 1, spillTableDesc, conf, !hasFilter(pos));
values.add((ArrayList<Object>) dummyObj[pos]);
dummyObjVectors[pos] = values;
@@ -319,7 +322,7 @@ public abstract class CommonJoinOperator
// e.g., the output columns does not contains the input table
RowContainer rc = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors.get((byte)pos),
- alias, joinCacheSize,spillTableDesc, conf,noOuterJoin);
+ alias, joinCacheSize,spillTableDesc, conf, !hasFilter(pos));
storage.put(pos, rc);
pos++;
@@ -443,14 +446,11 @@ transient boolean newGroupStarted = fals
private ArrayList<boolean[]> joinObjectsLeftOuterJoin(
ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left,
+ ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
boolean newObjNull) {
// newObj is null if is already null or
// if the row corresponding to the left alias does not pass through filter
- int filterIndex = joinValues.get(order[left]).size();
- if(filterIndex < intObj.getObjs()[left].size()) {
- newObjNull = newObjNull || ((BooleanWritable) (intObj.getObjs()[left].get(filterIndex))).get();
- }
+ newObjNull |= isLeftFiltered(left, right, intObj.getObjs()[left]);
Iterator<boolean[]> nullsIter = inputNulls.iterator();
while (nullsIter.hasNext()) {
@@ -470,7 +470,7 @@ transient boolean newGroupStarted = fals
private ArrayList<boolean[]> joinObjectsRightOuterJoin(
ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left,
+ ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
boolean newObjNull, boolean firstRow) {
if (newObjNull) {
return resNulls;
@@ -498,7 +498,7 @@ transient boolean newGroupStarted = fals
}
// if the row does not pass through filter, all old Objects are null
- if (((BooleanWritable)newObj.get(newObj.size()-1)).get()) {
+ if (isRightFiltered(left, right, newObj)) {
allOldObjsNull = true;
}
nullsIter = inputNulls.iterator();
@@ -526,7 +526,7 @@ transient boolean newGroupStarted = fals
private ArrayList<boolean[]> joinObjectsFullOuterJoin(
ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
- ArrayList<Object> newObj, IntermediateObject intObj, int left,
+ ArrayList<Object> newObj, IntermediateObject intObj, int left, int right,
boolean newObjNull, boolean firstRow) {
if (newObjNull) {
Iterator<boolean[]> nullsIter = inputNulls.iterator();
@@ -562,7 +562,7 @@ transient boolean newGroupStarted = fals
}
// if the row does not pass through filter, all old Objects are null
- if (((BooleanWritable)newObj.get(newObj.size()-1)).get()) {
+ if (isRightFiltered(left, right, newObj)) {
allOldObjsNull = true;
}
boolean rhsPreserved = false;
@@ -572,9 +572,8 @@ transient boolean newGroupStarted = fals
boolean[] oldNulls = nullsIter.next();
// old obj is null even if the row corresponding to the left alias
// does not pass through filter
- boolean oldObjNull = oldNulls[left] || ((BooleanWritable)
- (intObj.getObjs()[left].get(joinValues.get(order[left]).size()))).get()
- || allOldObjsNull;
+ boolean oldObjNull = oldNulls[left] || allOldObjsNull
+ || isLeftFiltered(left, right, intObj.getObjs()[left]);
if (!oldObjNull) {
boolean[] newNulls = new boolean[intObj.getCurSize()];
copyOldArray(oldNulls, newNulls);
@@ -623,6 +622,7 @@ transient boolean newGroupStarted = fals
}
int left = condn[joinPos - 1].getLeft();
+ int right = condn[joinPos - 1].getRight();
int type = condn[joinPos - 1].getType();
// process all nulls for RIGHT and FULL OUTER JOINS
@@ -646,17 +646,17 @@ transient boolean newGroupStarted = fals
newObjNull);
} else if (type == JoinDesc.LEFT_OUTER_JOIN) {
return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj,
- left, newObjNull);
+ left, right, newObjNull);
} else if (type == JoinDesc.RIGHT_OUTER_JOIN) {
return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj,
- left, newObjNull, firstRow);
+ left, right, newObjNull, firstRow);
} else if (type == JoinDesc.LEFT_SEMI_JOIN) {
return joinObjectsLeftSemiJoin(resNulls, inputNulls, newObj, intObj,
left, newObjNull);
}
assert (type == JoinDesc.FULL_OUTER_JOIN);
- return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left,
+ return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left, right,
newObjNull, firstRow);
}
@@ -821,19 +821,14 @@ transient boolean newGroupStarted = fals
hasEmpty = true;
alw.add((ArrayList<Object>) dummyObj[i]);
} else if (!hasEmpty && alw.size() == 1) {
- ArrayList<Object> row = alw.first();
- int numValues = joinValues.get(alias).size();
- if (row == dummyObj[alias]
- || (row.size() > numValues && ((BooleanWritable) (row.get(numValues))).get())) {
+ if (hasAnyFiltered(alias, alw.first())) {
hasEmpty = true;
}
} else {
mayHasMoreThanOne = true;
if (!hasEmpty) {
- int numValues = joinValues.get(alias).size();
for (ArrayList<Object> row = alw.first(); row != null; row = alw.next()) {
- if (row == dummyObj[alias]
- || (row.size() > numValues && ((BooleanWritable) (row.get(numValues))).get())) {
+ if (hasAnyFiltered(alias, row)) {
hasEmpty = true;
break;
}
@@ -860,6 +855,34 @@ transient boolean newGroupStarted = fals
}
}
+ // returns filter result of left object by filters associated with right alias
+ private boolean isLeftFiltered(int left, int right, List<Object> leftObj) {
+ if (joinValues.get(order[left]).size() < leftObj.size()) {
+ ByteWritable filter = (ByteWritable) leftObj.get(leftObj.size() - 1);
+ return JoinUtil.isFiltered(filter.get(), right);
+ }
+ return false;
+ }
+
+ // returns filter result of right object by filters associated with left alias
+ private boolean isRightFiltered(int left, int right, List<Object> rightObj) {
+ if (joinValues.get(order[right]).size() < rightObj.size()) {
+ ByteWritable filter = (ByteWritable) rightObj.get(rightObj.size() - 1);
+ return JoinUtil.isFiltered(filter.get(), left);
+ }
+ return false;
+ }
+
+ // returns object has any filtered tag
+ private boolean hasAnyFiltered(int alias, List<Object> row) {
+ return row == dummyObj[alias] ||
+ hasFilter(alias) && JoinUtil.hasAnyFiltered(((ByteWritable) row.get(row.size() - 1)).get());
+ }
+
+ protected final boolean hasFilter(int alias) {
+ return filterMap != null && filterMap[alias] != null;
+ }
+
protected void reportProgress() {
// Send some status periodically
countAfterReport++;
@@ -872,25 +895,6 @@ transient boolean newGroupStarted = fals
}
/**
- * Returns true if the row does not pass through filters.
- */
- protected static Boolean isFiltered(Object row,
- List<ExprNodeEvaluator> filters, List<ObjectInspector> ois)
- throws HiveException {
- // apply join filters on the row.
- Boolean ret = false;
- for (int j = 0; j < filters.size(); j++) {
- Object condition = filters.get(j).evaluate(row);
- ret = (Boolean) ((PrimitiveObjectInspector)
- ois.get(j)).getPrimitiveJavaObject(condition);
- if (ret == null || !ret) {
- return true;
- }
- }
- return false;
- }
-
- /**
* All done.
*
*/
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Tue Sep 25 17:32:09 2012
@@ -48,7 +48,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.util.ReflectionUtils;
@@ -86,6 +85,8 @@ public class HashTableSinkOperator exten
*/
protected transient Map<Byte, List<ExprNodeEvaluator>> joinFilters;
+ protected transient int[][] filterMap;
+
protected transient int numAliases; // number of aliases
/**
* The expressions for join outputs.
@@ -197,6 +198,7 @@ public class HashTableSinkOperator exten
totalSz = 0;
noOuterJoin = conf.isNoOuterJoin();
+ filterMap = conf.getFilterMap();
// process join keys
joinKeys = new HashMap<Byte, List<ExprNodeEvaluator>>();
@@ -228,10 +230,12 @@ public class HashTableSinkOperator exten
if (alias == posBigTableAlias) {
continue;
}
- ArrayList<ObjectInspector> rcOIs = new ArrayList<ObjectInspector>();
- rcOIs.addAll(joinValuesObjectInspectors.get(alias));
- // for each alias, add object inspector for boolean as the last element
- rcOIs.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
+ List<ObjectInspector> rcOIs = joinValuesObjectInspectors.get(alias);
+ if (filterMap != null && filterMap[alias] != null) {
+ // for each alias, add object inspector for filter tag as the last element
+ rcOIs = new ArrayList<ObjectInspector>(rcOIs);
+ rcOIs.add(PrimitiveObjectInspectorFactory.writableByteObjectInspector);
+ }
rowContainerObjectInspectors.put(alias, rcOIs);
}
rowContainerStandardObjectInspectors = getStandardObjectInspectors(rowContainerObjectInspectors);
@@ -318,7 +322,7 @@ public class HashTableSinkOperator exten
Object[] value = JoinUtil.computeMapJoinValues(row, joinValues.get(alias),
joinValuesObjectInspectors.get(alias), joinFilters.get(alias), joinFilterObjectInspectors
- .get(alias), noOuterJoin);
+ .get(alias), filterMap == null ? null : filterMap[alias]);
HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue> hashTable = mapJoinTables
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Tue Sep 25 17:32:09 2012
@@ -83,7 +83,8 @@ public class JoinOperator extends Common
ArrayList<Object> nr = JoinUtil.computeValues(row, joinValues.get(alias),
joinValuesObjectInspectors.get(alias), joinFilters.get(alias),
- joinFilterObjectInspectors.get(alias), noOuterJoin);
+ joinFilterObjectInspectors.get(alias),
+ filterMap == null ? null : filterMap[alias]);
if (handleSkewJoin) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Tue Sep 25 17:32:09 2012
@@ -45,7 +45,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
@@ -207,14 +207,14 @@ public class JoinUtil {
public static Object[] computeMapJoinValues(Object row,
List<ExprNodeEvaluator> valueFields, List<ObjectInspector> valueFieldsOI,
List<ExprNodeEvaluator> filters, List<ObjectInspector> filtersOI,
- boolean noOuterJoin) throws HiveException {
+ int[] filterMap) throws HiveException {
// Compute the keys
Object[] nr;
- if (!noOuterJoin) {
+ if (filterMap != null) {
nr = new Object[valueFields.size()+1];
// add whether the row is filtered or not.
- nr[valueFields.size()] = new BooleanWritable(isFiltered(row, filters, filtersOI));
+ nr[valueFields.size()] = new ByteWritable(isFiltered(row, filters, filtersOI, filterMap));
}else{
nr = new Object[valueFields.size()];
}
@@ -235,7 +235,7 @@ public class JoinUtil {
public static ArrayList<Object> computeValues(Object row,
List<ExprNodeEvaluator> valueFields, List<ObjectInspector> valueFieldsOI,
List<ExprNodeEvaluator> filters, List<ObjectInspector> filtersOI,
- boolean noOuterJoin) throws HiveException {
+ int[] filterMap) throws HiveException {
// Compute the values
ArrayList<Object> nr = new ArrayList<Object>(valueFields.size());
@@ -244,54 +244,77 @@ public class JoinUtil {
.evaluate(row), valueFieldsOI.get(i),
ObjectInspectorCopyOption.WRITABLE));
}
- if (!noOuterJoin) {
+ if (filterMap != null) {
// add whether the row is filtered or not.
- nr.add(new BooleanWritable(isFiltered(row, filters, filtersOI)));
+ nr.add(new ByteWritable(isFiltered(row, filters, filtersOI, filterMap)));
}
return nr;
}
+
+ private static final byte[] MASKS = new byte[]
+ {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, (byte) 0x80};
+
/**
* Returns true if the row does not pass through filters.
*/
- protected static Boolean isFiltered(Object row,
- List<ExprNodeEvaluator> filters, List<ObjectInspector> ois)
- throws HiveException {
+ protected static byte isFiltered(Object row, List<ExprNodeEvaluator> filters,
+ List<ObjectInspector> ois, int[] filterMap) throws HiveException {
// apply join filters on the row.
- Boolean ret = false;
- for (int j = 0; j < filters.size(); j++) {
- Object condition = filters.get(j).evaluate(row);
- ret = (Boolean) ((PrimitiveObjectInspector)
- ois.get(j)).getPrimitiveJavaObject(condition);
- if (ret == null || !ret) {
- return true;
+ byte ret = 0;
+ int j = 0;
+ for (int i = 0; i < filterMap.length; i += 2) {
+ int tag = filterMap[i];
+ int length = filterMap[i + 1];
+
+ boolean passed = true;
+ for (; length > 0; length--, j++) {
+ if (passed) {
+ Object condition = filters.get(j).evaluate(row);
+ Boolean result = (Boolean) ((PrimitiveObjectInspector)
+ ois.get(j)).getPrimitiveJavaObject(condition);
+ if (result == null || !result) {
+ passed = false;
+ }
+ }
+ }
+ if (!passed) {
+ ret |= MASKS[tag];
}
}
- return false;
+ return ret;
+ }
+
+ protected static boolean isFiltered(byte filter, int tag) {
+ return (filter & MASKS[tag]) != 0;
+ }
+
+ protected static boolean hasAnyFiltered(byte tag) {
+ return tag != 0;
}
public static TableDesc getSpillTableDesc(Byte alias,
Map<Byte, TableDesc> spillTableDesc,JoinDesc conf,
- boolean noOuterJoin) {
+ boolean noFilter) {
if (spillTableDesc == null || spillTableDesc.size() == 0) {
- spillTableDesc = initSpillTables(conf,noOuterJoin);
+ spillTableDesc = initSpillTables(conf,noFilter);
}
return spillTableDesc.get(alias);
}
public static Map<Byte, TableDesc> getSpillTableDesc(
Map<Byte, TableDesc> spillTableDesc,JoinDesc conf,
- boolean noOuterJoin) {
+ boolean noFilter) {
if (spillTableDesc == null) {
- spillTableDesc = initSpillTables(conf,noOuterJoin);
+ spillTableDesc = initSpillTables(conf,noFilter);
}
return spillTableDesc;
}
public static SerDe getSpillSerDe(byte alias,
Map<Byte, TableDesc> spillTableDesc,JoinDesc conf,
- boolean noOuterJoin) {
- TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noOuterJoin);
+ boolean noFilter) {
+ TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter);
if (desc == null) {
return null;
}
@@ -306,7 +329,7 @@ public class JoinUtil {
return sd;
}
- public static Map<Byte, TableDesc> initSpillTables(JoinDesc conf,boolean noOuterJoin) {
+ public static Map<Byte, TableDesc> initSpillTables(JoinDesc conf, boolean noFilter) {
Map<Byte, List<ExprNodeDesc>> exprs = conf.getExprs();
Map<Byte, TableDesc> spillTableDesc = new HashMap<Byte, TableDesc>(exprs.size());
for (int tag = 0; tag < exprs.size(); tag++) {
@@ -325,10 +348,10 @@ public class JoinUtil {
colTypes.append(valueCols.get(k).getTypeString());
colTypes.append(',');
}
- if (!noOuterJoin) {
+ if (!noFilter) {
colNames.append("filtered");
colNames.append(',');
- colTypes.append(TypeInfoFactory.booleanTypeInfo.getTypeName());
+ colTypes.append(TypeInfoFactory.byteTypeInfo.getTypeName());
colTypes.append(',');
}
// remove the last ','
@@ -352,11 +375,10 @@ public class JoinUtil {
public static RowContainer getRowContainer(Configuration hconf,
List<ObjectInspector> structFieldObjectInspectors,
Byte alias,int containerSize, Map<Byte, TableDesc> spillTableDesc,
- JoinDesc conf,boolean noOuterJoin) throws HiveException {
+ JoinDesc conf,boolean noFilter) throws HiveException {
- TableDesc tblDesc = JoinUtil.getSpillTableDesc(alias,spillTableDesc,conf, noOuterJoin);
- SerDe serde = JoinUtil.getSpillSerDe(alias, spillTableDesc, conf,
- noOuterJoin);
+ TableDesc tblDesc = JoinUtil.getSpillTableDesc(alias,spillTableDesc,conf, noFilter);
+ SerDe serde = JoinUtil.getSpillSerDe(alias, spillTableDesc, conf, noFilter);
if (serde == null) {
containerSize = -1;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Tue Sep 25 17:32:09 2012
@@ -238,7 +238,7 @@ public class MapJoinOperator extends Abs
joinKeysObjectInspectors.get(alias));
ArrayList<Object> value = JoinUtil.computeValues(row, joinValues.get(alias),
joinValuesObjectInspectors.get(alias), joinFilters.get(alias), joinFilterObjectInspectors
- .get(alias), noOuterJoin);
+ .get(alias), filterMap == null ? null : filterMap[alias]);
// Add the value to the ArrayList
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Tue Sep 25 17:32:09 2012
@@ -116,11 +116,11 @@ public class SMBMapJoinOperator extends
for (Byte alias : order) {
RowContainer rc = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors.get(storePos),
- alias, bucketSize,spillTableDesc, conf,noOuterJoin);
+ alias, bucketSize,spillTableDesc, conf, !hasFilter(storePos));
nextGroupStorage[storePos] = rc;
RowContainer candidateRC = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors.get((byte)storePos),
- alias,bucketSize,spillTableDesc, conf,noOuterJoin);
+ alias,bucketSize,spillTableDesc, conf, !hasFilter(storePos));
candidateStorage[alias] = candidateRC;
storePos++;
}
@@ -237,7 +237,8 @@ public class SMBMapJoinOperator extends
joinKeysObjectInspectors.get(alias));
ArrayList<Object> value = JoinUtil.computeValues(row, joinValues.get(alias),
joinValuesObjectInspectors.get(alias), joinFilters.get(alias),
- joinFilterObjectInspectors.get(alias), noOuterJoin);
+ joinFilterObjectInspectors.get(alias),
+ filterMap == null ? null : filterMap[alias]);
//have we reached a new key group?
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java Tue Sep 25 17:32:09 2012
@@ -116,6 +116,7 @@ public class SkewJoinHandler {
bigKeysExistingMap = new HashMap<Byte, Boolean>(numAliases);
taskId = Utilities.getTaskId(hconf);
+ int[][] filterMap = desc.getFilterMap();
for (int i = 0; i < numAliases; i++) {
Byte alias = conf.getTagOrder()[i];
List<ObjectInspector> skewTableKeyInspectors = new ArrayList<ObjectInspector>();
@@ -145,7 +146,9 @@ public class SkewJoinHandler {
break;
}
- TableDesc valTblDesc = JoinUtil.getSpillTableDesc(alias,joinOp.spillTableDesc,conf,noOuterJoin);
+ boolean hasFilter = filterMap != null && filterMap[i] != null;
+ TableDesc valTblDesc = JoinUtil.getSpillTableDesc(alias,
+ joinOp.spillTableDesc, conf, !hasFilter);
List<String> valColNames = new ArrayList<String>();
if (valTblDesc != null) {
valColNames = Utilities.getColumnNames(valTblDesc.getProperties());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Tue Sep 25 17:32:09 2012
@@ -363,8 +363,8 @@ public class MapJoinProcessor implements
valueExprMap.put(Byte.valueOf((byte) pos), values);
}
- Map<Byte, List<ExprNodeDesc>> filterMap = desc.getFilters();
- for (Map.Entry<Byte, List<ExprNodeDesc>> entry : filterMap.entrySet()) {
+ Map<Byte, List<ExprNodeDesc>> filters = desc.getFilters();
+ for (Map.Entry<Byte, List<ExprNodeDesc>> entry : filters.entrySet()) {
Byte srcAlias = entry.getKey();
List<ExprNodeDesc> columnDescList = entry.getValue();
@@ -411,6 +411,7 @@ public class MapJoinProcessor implements
List<TableDesc> valueTableDescs = new ArrayList<TableDesc>();
List<TableDesc> valueFiltedTableDescs = new ArrayList<TableDesc>();
+ int[][] filterMap = desc.getFilterMap();
for (pos = 0; pos < newParentOps.size(); pos++) {
List<ExprNodeDesc> valueCols = valueExprMap.get(Byte.valueOf((byte) pos));
int length = valueCols.size();
@@ -419,11 +420,9 @@ public class MapJoinProcessor implements
for (int i = 0; i < length; i++) {
valueFilteredCols.add(valueCols.get(i).clone());
}
- List<ExprNodeDesc> valueFilters = filterMap.get(Byte.valueOf((byte) pos));
-
- if (valueFilters != null && valueFilters.size() != 0 && pos != mapJoinPos) {
+ if (filterMap != null && filterMap[pos] != null && pos != mapJoinPos) {
ExprNodeColumnDesc isFilterDesc = new ExprNodeColumnDesc(TypeInfoFactory
- .getPrimitiveTypeInfo(Constants.BOOLEAN_TYPE_NAME), "filter", "filter", false);
+ .getPrimitiveTypeInfo(Constants.TINYINT_TYPE_NAME), "filter", "filter", false);
valueFilteredCols.add(isFilterDesc);
}
@@ -452,9 +451,10 @@ public class MapJoinProcessor implements
}
MapJoinDesc mapJoinDescriptor = new MapJoinDesc(keyExprMap, keyTableDesc, valueExprMap,
valueTableDescs, valueFiltedTableDescs, outputColumnNames, mapJoinPos, joinCondns,
- filterMap, op.getConf().getNoOuterJoin(), dumpFilePrefix);
+ filters, op.getConf().getNoOuterJoin(), dumpFilePrefix);
mapJoinDescriptor.setTagOrder(tagOrder);
mapJoinDescriptor.setNullSafes(desc.getNullSafes());
+ mapJoinDescriptor.setFilterMap(desc.getFilterMap());
MapJoinOperator mapJoinOp = (MapJoinOperator) OperatorFactory.getAndMakeChild(
mapJoinDescriptor, new RowSchema(outputRS.getColumnInfos()), newPar);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java Tue Sep 25 17:32:09 2012
@@ -53,6 +53,9 @@ public class QBJoinTree implements Seria
// filters
private ArrayList<ArrayList<ASTNode>> filters;
+ // outerjoin-pos = other-pos:filter-len, other-pos:filter-len, ...
+ private int[][] filterMap;
+
// filters for pushing
private ArrayList<ArrayList<ASTNode>> filtersForPushing;
@@ -305,4 +308,16 @@ public class QBJoinTree implements Seria
public void setNullSafes(ArrayList<Boolean> nullSafes) {
this.nullsafes = nullSafes;
}
+
+ public void addFilterMapping(int outer, int target, int length) {
+ filterMap[outer] = new int[] { target, length };
+ }
+
+ public int[][] getFilterMap() {
+ return filterMap;
+ }
+
+ public void setFilterMap(int[][] filterMap) {
+ this.filterMap = filterMap;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Sep 25 17:32:09 2012
@@ -1247,9 +1247,9 @@ public class SemanticAnalyzer extends Ba
}
}
- private void populateAliases(ArrayList<String> leftAliases,
- ArrayList<String> rightAliases, ASTNode condn, QBJoinTree joinTree,
- ArrayList<String> leftSrc) throws SemanticException {
+ private void populateAliases(List<String> leftAliases,
+ List<String> rightAliases, ASTNode condn, QBJoinTree joinTree,
+ List<String> leftSrc) throws SemanticException {
if ((leftAliases.size() != 0) && (rightAliases.size() != 0)) {
throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1
.getMsg(condn));
@@ -1271,6 +1271,24 @@ public class SemanticAnalyzer extends Ba
}
}
+ private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond, List<String> leftSrc)
+ throws SemanticException {
+ if (joinCond == null) {
+ return;
+ }
+ JoinCond cond = joinTree.getJoinCond()[0];
+
+ JoinType type = cond.getJoinType();
+ parseJoinCondition(joinTree, joinCond, leftSrc, type);
+
+ List<ArrayList<ASTNode>> filters = joinTree.getFilters();
+ if (type == JoinType.LEFTOUTER || type == JoinType.FULLOUTER) {
+ joinTree.addFilterMapping(cond.getLeft(), cond.getRight(), filters.get(0).size());
+ }
+ if (type == JoinType.RIGHTOUTER || type == JoinType.FULLOUTER) {
+ joinTree.addFilterMapping(cond.getRight(), cond.getLeft(), filters.get(1).size());
+ }
+ }
/**
* Parse the join condition. If the condition is a join condition, throw an
* error if it is not an equality. Otherwise, break it into left and right
@@ -1291,20 +1309,19 @@ public class SemanticAnalyzer extends Ba
* @throws SemanticException
*/
private void parseJoinCondition(QBJoinTree joinTree, ASTNode joinCond,
- ArrayList<String> leftSrc) throws SemanticException {
+ List<String> leftSrc, JoinType type) throws SemanticException {
if (joinCond == null) {
return;
}
- JoinType type = joinTree.getJoinCond()[0].getJoinType();
switch (joinCond.getToken().getType()) {
case HiveParser.KW_OR:
throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_3
.getMsg(joinCond));
case HiveParser.KW_AND:
- parseJoinCondition(joinTree, (ASTNode) joinCond.getChild(0), leftSrc);
- parseJoinCondition(joinTree, (ASTNode) joinCond.getChild(1), leftSrc);
+ parseJoinCondition(joinTree, (ASTNode) joinCond.getChild(0), leftSrc, type);
+ parseJoinCondition(joinTree, (ASTNode) joinCond.getChild(1), leftSrc, type);
break;
case HiveParser.EQUAL_NS:
@@ -4949,6 +4966,8 @@ public class SemanticAnalyzer extends Ba
JoinDesc desc = new JoinDesc(exprMap, outputColumnNames,
join.getNoOuterJoin(), joinCondns, filterMap);
desc.setReversedExprs(reversedExprs);
+ desc.setFilterMap(join.getFilterMap());
+
JoinOperator joinOp = (JoinOperator) OperatorFactory.getAndMakeChild(desc,
new RowSchema(outputRS.getColumnInfos()), rightOps);
joinOp.setColumnExprMap(colExprMap);
@@ -5473,6 +5492,7 @@ public class SemanticAnalyzer extends Ba
filters.add(new ArrayList<ASTNode>());
filters.add(new ArrayList<ASTNode>());
joinTree.setFilters(filters);
+ joinTree.setFilterMap(new int[2][]);
ArrayList<ArrayList<ASTNode>> filtersForPushing =
new ArrayList<ArrayList<ASTNode>>();
@@ -5590,6 +5610,33 @@ public class SemanticAnalyzer extends Ba
filterPos.addAll(node.getFilters().get(0));
}
+ int[][] nmap = node.getFilterMap();
+ int[][] tmap = target.getFilterMap();
+ int[][] newmap = new int[tmap.length + nmap.length - 1][];
+
+ for (int[] mapping : nmap) {
+ if (mapping != null) {
+ for (int i = 0; i < mapping.length; i+=2) {
+ if (pos > 0 || mapping[i] > 0) {
+ mapping[i] += trgtRightAliases.length;
+ }
+ }
+ }
+ }
+ if (nmap[0] != null) {
+ if (tmap[pos] == null) {
+ tmap[pos] = nmap[0];
+ } else {
+ int[] appended = new int[tmap[pos].length + nmap[0].length];
+ System.arraycopy(tmap[pos], 0, appended, 0, tmap[pos].length);
+ System.arraycopy(nmap[0], 0, appended, tmap[pos].length, nmap[0].length);
+ tmap[pos] = appended;
+ }
+ }
+ System.arraycopy(tmap, 0, newmap, 0, tmap.length);
+ System.arraycopy(nmap, 1, newmap, tmap.length, nmap.length - 1);
+ target.setFilterMap(newmap);
+
ArrayList<ArrayList<ASTNode>> filter = target.getFiltersForPushing();
for (int i = 0; i < nodeRightAliases.length; i++) {
filter.add(node.getFiltersForPushing().get(i + 1));
@@ -5695,7 +5742,13 @@ public class SemanticAnalyzer extends Ba
if (target == null) {
return false;
}
-
+ if (!node.getNoOuterJoin() || !target.getNoOuterJoin()) {
+ // todo 8 way could be not enough number
+ if (node.getRightAliases().length + node.getRightAliases().length + 1 >= 8) {
+ LOG.info(ErrorMsg.JOINNODE_OUTERJOIN_MORETHAN_8);
+ return false;
+ }
+ }
int res = findMergePos(node, target);
if (res != -1) {
mergeJoins(qb, parent, node, target, res);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java Tue Sep 25 17:32:09 2012
@@ -20,8 +20,10 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -49,6 +51,9 @@ public class HashTableSinkDesc extends J
// alias to filter mapping
private Map<Byte, List<ExprNodeDesc>> filters;
+ // outerjoin-pos = other-pos:filter-len, other-pos:filter-len, ...
+ private int[][] filterMap;
+
// used for create joinOutputObjectInspector
protected List<String> outputColumnNames;
@@ -98,6 +103,7 @@ public class HashTableSinkDesc extends J
this.smallKeysDirMap = clone.getSmallKeysDirMap();
this.tagOrder = clone.getTagOrder();
this.filters = clone.getFilters();
+ this.filterMap = clone.getFilterMap();
this.keys = clone.getKeys();
this.keyTblDesc = clone.getKeyTblDesc();
@@ -285,6 +291,21 @@ public class HashTableSinkDesc extends J
this.keyTableDesc = keyTableDesc;
}
+ @Override
+ public int[][] getFilterMap() {
+ return filterMap;
+ }
+
+ @Override
+ public void setFilterMap(int[][] filterMap) {
+ this.filterMap = filterMap;
+ }
+
+ @Override
+ @Explain(displayName = "filter mappings", normalExplain = false)
+ public Map<Integer, String> getFilterMapString() {
+ return toCompactString(filterMap);
+ }
public Map<Byte, List<Integer>> getRetainList() {
return retainList;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java Tue Sep 25 17:32:09 2012
@@ -53,6 +53,16 @@ public class JoinDesc extends AbstractOp
// alias to filter mapping
private Map<Byte, List<ExprNodeDesc>> filters;
+ // pos of outer join alias=<pos of other alias:num of filters on outer join alias>xn
+ // for example,
+ // a left outer join b on a.k=b.k AND a.k>5 full outer join c on a.k=c.k AND a.k>10 AND c.k>20
+ //
+ // That means on a(pos=0), there are overlapped filters associated with b(pos=1) and c(pos=2).
+ // (a)b has one filter on a (a.k>5) and (a)c also has one filter on a (a.k>10),
+ // making filter map for a as 0=1:1:2:1.
+ // C also has one outer join filter associated with A(c.k>20), which is making 2=0:1
+ private int[][] filterMap;
+
// key index to nullsafe join flag
private boolean[] nullsafes;
@@ -166,6 +176,7 @@ public class JoinDesc extends AbstractOp
this.smallKeysDirMap = clone.smallKeysDirMap;
this.tagOrder = clone.tagOrder;
this.filters = clone.filters;
+ this.filterMap = clone.filterMap;
}
public Map<Byte, List<ExprNodeDesc>> getExprs() {
@@ -435,4 +446,60 @@ public class JoinDesc extends AbstractOp
}
return hasNS ? Arrays.toString(nullsafes) : null;
}
+
+ public int[][] getFilterMap() {
+ return filterMap;
+ }
+
+ public void setFilterMap(int[][] filterMap) {
+ this.filterMap = filterMap;
+ }
+
+ @Explain(displayName = "filter mappings", normalExplain = false)
+ public Map<Integer, String> getFilterMapString() {
+ return toCompactString(filterMap);
+ }
+
+ protected Map<Integer, String> toCompactString(int[][] filterMap) {
+ if (filterMap == null) {
+ return null;
+ }
+ filterMap = compactFilter(filterMap);
+ Map<Integer, String> result = new LinkedHashMap<Integer, String>();
+ for (int i = 0 ; i < filterMap.length; i++) {
+ if (filterMap[i] == null) {
+ continue;
+ }
+ result.put(i, Arrays.toString(filterMap[i]));
+ }
+ return result.isEmpty() ? null : result;
+ }
+
+ // remove filterMap for outer alias if filter is not exist on that
+ private int[][] compactFilter(int[][] filterMap) {
+ if (filterMap == null) {
+ return null;
+ }
+ for (int i = 0; i < filterMap.length; i++) {
+ if (filterMap[i] != null) {
+ boolean noFilter = true;
+ // join positions for even index, filter lengths for odd index
+ for (int j = 1; j < filterMap[i].length; j += 2) {
+ if (filterMap[i][j] > 0) {
+ noFilter = false;
+ break;
+ }
+ }
+ if (noFilter) {
+ filterMap[i] = null;
+ }
+ }
+ }
+ for (int[] mapping : filterMap) {
+ if (mapping != null) {
+ return filterMap;
+ }
+ }
+ return null;
+ }
}
Added: hive/trunk/ql/src/test/queries/clientpositive/join_filters_overlap.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/join_filters_overlap.q?rev=1390010&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/join_filters_overlap.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/join_filters_overlap.q Tue Sep 25 17:32:09 2012
@@ -0,0 +1,27 @@
+-- HIVE-3411 Filter predicates on outer join overlapped on single alias is not handled properly
+
+create table a as SELECT 100 as key, a.value as value FROM src LATERAL VIEW explode(array(40, 50, 60)) a as value limit 3;
+
+-- overlap on a
+explain extended select * from a left outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (a.key=c.key AND a.value=60 AND c.value=60);
+select * from a left outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (a.key=c.key AND a.value=60 AND c.value=60);
+select /*+ MAPJOIN(b,c)*/ * from a left outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (a.key=c.key AND a.value=60 AND c.value=60);
+
+-- overlap on b
+explain extended select * from a right outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (b.key=c.key AND b.value=60 AND c.value=60);
+select * from a right outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (b.key=c.key AND b.value=60 AND c.value=60);
+select /*+ MAPJOIN(a,c)*/ * from a right outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (b.key=c.key AND b.value=60 AND c.value=60);
+
+-- overlap on b with two filters for each
+explain extended select * from a right outer join a b on (a.key=b.key AND a.value=50 AND b.value=50 AND b.value>10) left outer join a c on (b.key=c.key AND b.value=60 AND b.value>20 AND c.value=60);
+select * from a right outer join a b on (a.key=b.key AND a.value=50 AND b.value=50 AND b.value>10) left outer join a c on (b.key=c.key AND b.value=60 AND b.value>20 AND c.value=60);
+select /*+ MAPJOIN(a,c)*/ * from a right outer join a b on (a.key=b.key AND a.value=50 AND b.value=50 AND b.value>10) left outer join a c on (b.key=c.key AND b.value=60 AND b.value>20 AND c.value=60);
+
+-- overlap on a, b
+explain extended select * from a full outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (b.key=c.key AND b.value=60 AND c.value=60) left outer join a d on (a.key=d.key AND a.value=40 AND d.value=40);
+select * from a full outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (b.key=c.key AND b.value=60 AND c.value=60) left outer join a d on (a.key=d.key AND a.value=40 AND d.value=40);
+
+-- triple overlap on a
+explain extended select * from a left outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (a.key=c.key AND a.value=60 AND c.value=60) left outer join a d on (a.key=d.key AND a.value=40 AND d.value=40);
+select * from a left outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (a.key=c.key AND a.value=60 AND c.value=60) left outer join a d on (a.key=d.key AND a.value=40 AND d.value=40);
+select /*+ MAPJOIN(b,c, d)*/ * from a left outer join a b on (a.key=b.key AND a.value=50 AND b.value=50) left outer join a c on (a.key=c.key AND a.value=60 AND c.value=60) left outer join a d on (a.key=d.key AND a.value=40 AND d.value=40);
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out?rev=1390010&r1=1390009&r2=1390010&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out Tue Sep 25 17:32:09 2012
@@ -4461,33 +4461,33 @@ POSTHOOK: query: SELECT * FROM src src1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
-NULL NULL 0 val_0 NULL NULL
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
NULL NULL 10 val_10 NULL NULL
NULL NULL 100 val_100 NULL NULL
NULL NULL 100 val_100 NULL NULL
@@ -4603,7 +4603,7 @@ NULL NULL 197 val_197 NULL NULL
NULL NULL 199 val_199 NULL NULL
NULL NULL 199 val_199 NULL NULL
NULL NULL 199 val_199 NULL NULL
-NULL NULL 2 val_2 NULL NULL
+NULL NULL 2 val_2 2 val_2
NULL NULL 20 val_20 NULL NULL
NULL NULL 200 val_200 NULL NULL
NULL NULL 200 val_200 NULL NULL
@@ -4813,7 +4813,7 @@ NULL NULL 397 val_397 NULL NULL
NULL NULL 397 val_397 NULL NULL
NULL NULL 399 val_399 NULL NULL
NULL NULL 399 val_399 NULL NULL
-NULL NULL 4 val_4 NULL NULL
+NULL NULL 4 val_4 4 val_4
NULL NULL 400 val_400 NULL NULL
NULL NULL 401 val_401 NULL NULL
NULL NULL 401 val_401 NULL NULL
@@ -4937,33 +4937,33 @@ NULL NULL 497 val_497 NULL NULL
NULL NULL 498 val_498 NULL NULL
NULL NULL 498 val_498 NULL NULL
NULL NULL 498 val_498 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
-NULL NULL 5 val_5 NULL NULL
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
NULL NULL 51 val_51 NULL NULL
NULL NULL 51 val_51 NULL NULL
NULL NULL 53 val_53 NULL NULL
@@ -4987,7 +4987,7 @@ NULL NULL 76 val_76 NULL NULL
NULL NULL 76 val_76 NULL NULL
NULL NULL 77 val_77 NULL NULL
NULL NULL 78 val_78 NULL NULL
-NULL NULL 8 val_8 NULL NULL
+NULL NULL 8 val_8 8 val_8
NULL NULL 80 val_80 NULL NULL
NULL NULL 82 val_82 NULL NULL
NULL NULL 83 val_83 NULL NULL
@@ -4997,7 +4997,7 @@ NULL NULL 84 val_84 NULL NULL
NULL NULL 85 val_85 NULL NULL
NULL NULL 86 val_86 NULL NULL
NULL NULL 87 val_87 NULL NULL
-NULL NULL 9 val_9 NULL NULL
+NULL NULL 9 val_9 9 val_9
NULL NULL 90 val_90 NULL NULL
NULL NULL 90 val_90 NULL NULL
NULL NULL 90 val_90 NULL NULL