You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rh...@apache.org on 2014/06/24 23:39:04 UTC
svn commit: r1605203 [1/4] - in /hive/branches/cbo: data/files/ ql/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/
ql/src/java/org/apache/hadoop/hive/ql/opt...
Author: rhbutani
Date: Tue Jun 24 21:39:03 2014
New Revision: 1605203
URL: http://svn.apache.org/r1605203
Log:
HIVE-7280 Commit CBO code from github repo to CBO branch (John Pullokkaran via Harish Butani)
Added:
hive/branches/cbo/data/files/cbo_t1.txt
hive/branches/cbo/data/files/cbo_t2.txt
hive/branches/cbo/data/files/cbo_t3.txt
hive/branches/cbo/data/files/cbo_t4.txt
hive/branches/cbo/data/files/cbo_t5.txt
hive/branches/cbo/data/files/cbo_t6.txt
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/JoinCondnTypeCheckProcFactory.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/JoinTypeCheckCtx.java
hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out
Removed:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/CostBasedOptimizer.java
Modified:
hive/branches/cbo/ql/pom.xml
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
Added: hive/branches/cbo/data/files/cbo_t1.txt
URL: http://svn.apache.org/viewvc/hive/branches/cbo/data/files/cbo_t1.txt?rev=1605203&view=auto
==============================================================================
--- hive/branches/cbo/data/files/cbo_t1.txt (added)
+++ hive/branches/cbo/data/files/cbo_t1.txt Tue Jun 24 21:39:03 2014
@@ -0,0 +1,20 @@
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+1,1,1,1,true
+ 1, 1,1,1,true
+ 1, 1,1,1,true
+ 1 , 1 ,1,1,true
+ 1 , 1 ,1,1,true
+1 ,1 ,1,1,true
+1 ,1 ,1,1,true
+1,1,1,1,false
+1,1,1,1,false
+null,null,null,null,null
+null,null,null,null,null
Added: hive/branches/cbo/data/files/cbo_t2.txt
URL: http://svn.apache.org/viewvc/hive/branches/cbo/data/files/cbo_t2.txt?rev=1605203&view=auto
==============================================================================
--- hive/branches/cbo/data/files/cbo_t2.txt (added)
+++ hive/branches/cbo/data/files/cbo_t2.txt Tue Jun 24 21:39:03 2014
@@ -0,0 +1,20 @@
+1,1,1,1,true
+2,2,2,2,true
+1,1,1,1,true
+2,2,2,2,true
+1,1,1,1,true
+2,2,2,2,true
+1,1,1,1,true
+2,2,2,2,true
+1,1,1,1,true
+2,2,2,2,true
+ 1, 1,1,1,true
+ 1, 1,1,1,true
+ 1 , 1 ,1,1,true
+ 1 , 1 ,1,1,true
+1 ,1 ,1,1,true
+1 ,1 ,1,1,true
+1,1,1,1,false
+1,1,1,1,false
+null,null,null,null,null
+null,null,null,null,null
Added: hive/branches/cbo/data/files/cbo_t3.txt
URL: http://svn.apache.org/viewvc/hive/branches/cbo/data/files/cbo_t3.txt?rev=1605203&view=auto
==============================================================================
--- hive/branches/cbo/data/files/cbo_t3.txt (added)
+++ hive/branches/cbo/data/files/cbo_t3.txt Tue Jun 24 21:39:03 2014
@@ -0,0 +1,20 @@
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+1,1,1,1,true
+ 1, 1,1,1,true
+ 1, 1,1,1,true
+ 1 , 1 ,1,1,true
+ 1 , 1 ,1,1,true
+1 ,1 ,1,1,true
+1 ,1 ,1,1,true
+1,1,1,1,false
+1,1,1,1,false
+null,null,null,null,null
+null,null,null,null,null
Added: hive/branches/cbo/data/files/cbo_t4.txt
URL: http://svn.apache.org/viewvc/hive/branches/cbo/data/files/cbo_t4.txt?rev=1605203&view=auto
==============================================================================
--- hive/branches/cbo/data/files/cbo_t4.txt (added)
+++ hive/branches/cbo/data/files/cbo_t4.txt Tue Jun 24 21:39:03 2014
@@ -0,0 +1,20 @@
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+4,4,4,4,true
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+4,4,4,4,true
+1,1,1,1,true
+2,2,2,2,true
+ 1, 1,1,1,true
+ 1, 1,1,1,true
+ 1 , 1 ,1,1,true
+ 1 , 1 ,1,1,true
+1 ,1 ,1,1,true
+1 ,1 ,1,1,true
+1,1,1,1,false
+1,1,1,1,false
+null,null,null,null,null
+null,null,null,null,null
Added: hive/branches/cbo/data/files/cbo_t5.txt
URL: http://svn.apache.org/viewvc/hive/branches/cbo/data/files/cbo_t5.txt?rev=1605203&view=auto
==============================================================================
--- hive/branches/cbo/data/files/cbo_t5.txt (added)
+++ hive/branches/cbo/data/files/cbo_t5.txt Tue Jun 24 21:39:03 2014
@@ -0,0 +1,20 @@
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+4,4,4,4,true
+5,5,5,5,true
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+4,4,4,4,true
+5,5,5,5,true
+ 1, 1,1,1,true
+ 1, 1,1,1,true
+ 1 , 1 ,1,1,true
+ 1 , 1 ,1,1,true
+1 ,1 ,1,1,true
+1 ,1 ,1,1,true
+1,1,1,1,false
+1,1,1,1,false
+null,null,null,null,null
+null,null,null,null,null
Added: hive/branches/cbo/data/files/cbo_t6.txt
URL: http://svn.apache.org/viewvc/hive/branches/cbo/data/files/cbo_t6.txt?rev=1605203&view=auto
==============================================================================
--- hive/branches/cbo/data/files/cbo_t6.txt (added)
+++ hive/branches/cbo/data/files/cbo_t6.txt Tue Jun 24 21:39:03 2014
@@ -0,0 +1,20 @@
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+4,4,4,4,true
+5,5,5,5,true
+6,6,6,6,true
+1,1,1,1,true
+2,2,2,2,true
+3,3,3,3,true
+4,4,4,4,true
+ 1, 1,1,1,true
+ 1, 1,1,1,true
+ 1 , 1 ,1,1,true
+ 1 , 1 ,1,1,true
+1 ,1 ,1,1,true
+1 ,1 ,1,1,true
+1,1,1,1,false
+1,1,1,1,false
+null,null,null,null,null
+null,null,null,null,null
Modified: hive/branches/cbo/ql/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/pom.xml?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/pom.xml (original)
+++ hive/branches/cbo/ql/pom.xml Tue Jun 24 21:39:03 2014
@@ -28,7 +28,7 @@
<name>Hive Query Language</name>
<properties>
- <optiq.version>0.5</optiq.version>
+ <optiq.version>0.7</optiq.version>
<hive.path.to.root>..</hive.path.to.root>
</properties>
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java Tue Jun 24 21:39:03 2014
@@ -50,7 +50,10 @@ public class QueryProperties {
private int noOfJoins = 0;
private int noOfOuterJoins = 0;
-
+
+ private boolean multiDestQuery;
+ private boolean filterWithSubQuery;
+
public boolean hasJoin() {
return (noOfJoins > 0);
}
@@ -157,6 +160,23 @@ public class QueryProperties {
this.hasMapGroupBy = hasMapGroupBy;
}
+ public boolean hasMultiDestQuery() {
+ return this.multiDestQuery;
+ }
+
+ public void setMultiDestQuery(boolean multiDestQuery) {
+ this.multiDestQuery = multiDestQuery;
+ }
+
+ public void setFilterWithSubQuery(boolean filterWithSubQuery) {
+ this.filterWithSubQuery = filterWithSubQuery;
+ }
+
+ public boolean hasFilterWithSubQuery() {
+ return this.filterWithSubQuery;
+ }
+
+
public void clear() {
hasJoin = false;
hasGroupBy = false;
@@ -176,5 +196,8 @@ public class QueryProperties {
noOfJoins = 0;
noOfOuterJoins = 0;
+
+ multiDestQuery = false;
+ filterWithSubQuery = false;
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java Tue Jun 24 21:39:03 2014
@@ -1,18 +1,30 @@
package org.apache.hadoop.hive.ql.optimizer.optiq;
+import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
import org.apache.hadoop.hive.ql.plan.Statistics;
+import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.eigenbase.rel.RelNode;
import org.eigenbase.rel.TableAccessRel;
import org.eigenbase.relopt.RelOptAbstractTable;
import org.eigenbase.relopt.RelOptSchema;
import org.eigenbase.reltype.RelDataType;
-//Fix Me: use table meta data and stats util to get stats
+/*
+ * Fix Me:
+ * 1. Column Pruning
+ * 2. Partition Pruning
+ * 3. Stats
+ */
+
public class RelOptHiveTable extends RelOptAbstractTable {
private final Table m_hiveTblMetadata;
private double m_rowCount = -1;
@@ -23,6 +35,7 @@ public class RelOptHiveTable extends Rel
Map<String, Integer> m_bucketingSortColMap;
Statistics m_hiveStats;
+ List<ColStatistics> m_hiveColStats = new ArrayList<ColStatistics>();
// NOTE: name here is the table alias which may or may not be the real name in
// metadata. Use
@@ -32,9 +45,21 @@ public class RelOptHiveTable extends Rel
Table hiveTblMetadata, Statistics stats) {
super(schema, name, rowType);
m_hiveTblMetadata = hiveTblMetadata;
- m_hiveStats = stats;
+ }
- m_rowCount = stats.getNumRows();
+ public RelOptHiveTable(RelOptSchema optiqSchema, String name, RelDataType rowType,
+ Table hiveTblMetadata, List<ColumnInfo> hiveSchema) {
+ super(optiqSchema, name, rowType);
+ m_hiveTblMetadata = hiveTblMetadata;
+
+ List<String> neededColumns = new ArrayList<String>();
+ for (ColumnInfo ci : hiveSchema) {
+ neededColumns.add(ci.getInternalName());
+ }
+
+ //TODO: Fix below two stats
+ m_hiveColStats = StatsUtils.getTableColumnStats(m_hiveTblMetadata, hiveSchema, neededColumns);
+ m_rowCount = StatsUtils.getNumRows(m_hiveTblMetadata);
}
@Override
@@ -64,4 +89,16 @@ public class RelOptHiveTable extends Rel
public Statistics getHiveStats() {
return m_hiveStats;
}
+
+ public List<ColStatistics> getColStat(List<Integer> projIndxLst) {
+ if (projIndxLst != null) {
+ List<ColStatistics> hiveColStatLst = new LinkedList<ColStatistics>();
+ for (Integer i : projIndxLst) {
+ hiveColStatLst.add(m_hiveColStats.get(i));
+ }
+ return hiveColStatLst;
+ } else {
+ return m_hiveColStats;
+ }
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/cost/HiveVolcanoPlanner.java Tue Jun 24 21:39:03 2014
@@ -25,7 +25,6 @@ public class HiveVolcanoPlanner extends
planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
if (ENABLE_COLLATION_TRAIT) {
planner.addRelTraitDef(RelCollationTraitDef.INSTANCE);
- planner.registerAbstractRelationalRules();
}
return planner;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveJoinRel.java Tue Jun 24 21:39:03 2014
@@ -17,6 +17,8 @@ import org.eigenbase.relopt.RelOptCost;
import org.eigenbase.relopt.RelOptPlanner;
import org.eigenbase.relopt.RelOptUtil;
import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeField;
import org.eigenbase.rex.RexNode;
//TODO: Should we convert MultiJoin to be a child of HiveJoinRelBase
@@ -34,15 +36,16 @@ public class HiveJoinRel extends JoinRel
NONE, LEFT_RELATION, RIGHT_RELATION
}
+ private final boolean m_leftSemiJoin;
private final JoinAlgorithm m_joinAlgorithm;
private MapJoinStreamingRelation m_mapJoinStreamingSide = MapJoinStreamingRelation.NONE;
public static HiveJoinRel getJoin(RelOptCluster cluster, RelNode left, RelNode right,
- RexNode condition, JoinRelType joinType) {
+ RexNode condition, JoinRelType joinType, boolean leftSemiJoin) {
try {
Set<String> variablesStopped = Collections.emptySet();
return new HiveJoinRel(cluster, null, left, right, condition, joinType, variablesStopped,
- JoinAlgorithm.NONE, null);
+ JoinAlgorithm.NONE, null, leftSemiJoin);
} catch (InvalidRelException e) {
throw new RuntimeException(e);
}
@@ -50,7 +53,7 @@ public class HiveJoinRel extends JoinRel
protected HiveJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right,
RexNode condition, JoinRelType joinType, Set<String> variablesStopped,
- JoinAlgorithm joinAlgo, MapJoinStreamingRelation streamingSideForMapJoin)
+ JoinAlgorithm joinAlgo, MapJoinStreamingRelation streamingSideForMapJoin, boolean leftSemiJoin)
throws InvalidRelException {
super(cluster, TraitsUtil.getJoinTraitSet(cluster, traits), left, right, condition, joinType,
variablesStopped);
@@ -58,13 +61,18 @@ public class HiveJoinRel extends JoinRel
final List<RexNode> leftKeys = new ArrayList<RexNode>();
final List<RexNode> rightKeys = new ArrayList<RexNode>();
List<Integer> filterNulls = new LinkedList<Integer>();
- RexNode remaining = RelOptUtil.splitJoinCondition(getSystemFieldList(), left, right, condition,
- leftKeys, rightKeys, filterNulls, null);
-
- if (!remaining.isAlwaysTrue()) {
- throw new InvalidRelException("EnumerableJoinRel only supports equi-join");
+ RexNode remaining = null;
+ if (condition != null) {
+ remaining = RelOptUtil.splitJoinCondition(getSystemFieldList(), left,
+ right, condition, leftKeys, rightKeys, filterNulls, null);
+
+ if (!remaining.isAlwaysTrue()) {
+ throw new InvalidRelException(
+ "EnumerableJoinRel only supports equi-join");
+ }
}
this.m_joinAlgorithm = joinAlgo;
+ m_leftSemiJoin = leftSemiJoin;
}
@Override
@@ -81,7 +89,7 @@ public class HiveJoinRel extends JoinRel
JoinAlgorithm joinalgo, MapJoinStreamingRelation streamingSide) {
try {
return new HiveJoinRel(getCluster(), traitSet, left, right, conditionExpr, joinType,
- variablesStopped, joinalgo, streamingSide);
+ variablesStopped, joinalgo, streamingSide, this.m_leftSemiJoin);
} catch (InvalidRelException e) {
// Semantic error not possible. Must be a bug. Convert to
// internal error.
@@ -93,8 +101,24 @@ public class HiveJoinRel extends JoinRel
return m_joinAlgorithm;
}
+ public boolean isLeftSemiJoin() {
+ return m_leftSemiJoin;
+ }
+
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner) {
return HiveCostUtil.computCardinalityBasedCost(this);
}
+
+ /**
+ * @return returns rowtype representing only the left join input
+ */
+ public RelDataType deriveRowType() {
+ if (m_leftSemiJoin) {
+ return deriveJoinRowType(left.getRowType(), null, JoinRelType.INNER,
+ getCluster().getTypeFactory(), null,
+ Collections.<RelDataTypeField> emptyList());
+ }
+ return super.deriveRowType();
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveSortRel.java Tue Jun 24 21:39:03 2014
@@ -2,7 +2,6 @@ package org.apache.hadoop.hive.ql.optimi
import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil;
import org.eigenbase.rel.RelCollation;
-import org.eigenbase.rel.RelCollationImpl;
import org.eigenbase.rel.RelNode;
import org.eigenbase.rel.SortRel;
import org.eigenbase.relopt.RelOptCluster;
@@ -10,12 +9,12 @@ import org.eigenbase.relopt.RelTraitSet;
import org.eigenbase.rex.RexNode;
public class HiveSortRel extends SortRel implements HiveRel {
-
+
public HiveSortRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode child,
RelCollation collation, RexNode offset, RexNode fetch) {
super(cluster, TraitsUtil.getSortTraitSet(cluster, traitSet, collation), child, collation,
offset, fetch);
-
+
assert getConvention() == child.getConvention();
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/reloperators/HiveTableScanRel.java Tue Jun 24 21:39:03 2014
@@ -1,6 +1,5 @@
package org.apache.hadoop.hive.ql.optimizer.optiq.reloperators;
-import java.util.LinkedList;
import java.util.List;
import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable;
@@ -15,7 +14,6 @@ import org.eigenbase.relopt.RelOptPlanne
import org.eigenbase.relopt.RelTraitSet;
import org.eigenbase.reltype.RelDataType;
-import com.google.common.collect.ImmutableList;
/**
* Relational expression representing a scan of a HiveDB collection.
@@ -26,7 +24,7 @@ import com.google.common.collect.Immutab
* </p>
*/
public class HiveTableScanRel extends TableAccessRelBase implements HiveRel {
- private final ImmutableList<ColStatistics> m_hiveColStat;
+ private List<ColStatistics> m_hiveColStat;
/**
* Creates a HiveTableScan.
@@ -44,14 +42,6 @@ public class HiveTableScanRel extends Ta
RelDataType rowtype) {
super(cluster, TraitsUtil.getTableScanTraitSet(cluster, traitSet, table, rowtype), table);
assert getConvention() == HiveRel.CONVENTION;
-
- ImmutableList.Builder<ColStatistics> b = new ImmutableList.Builder<ColStatistics>();
- for (String fN : rowtype.getFieldNames()) {
- ColStatistics cStat = table.getHiveStats().getColumnStatisticsForColumn(
- table.getName(), fN);
- b.add(cStat);
- }
- m_hiveColStat = b.build();
}
@Override
@@ -78,17 +68,8 @@ public class HiveTableScanRel extends Ta
public double getRows() {
return ((RelOptHiveTable) table).getRowCount();
}
-
+
public List<ColStatistics> getColStat(List<Integer> projIndxLst) {
- if (projIndxLst != null) {
- List<ColStatistics> hiveColStatLst = new LinkedList<ColStatistics>();
- for (Integer i : projIndxLst) {
- hiveColStatLst.add(m_hiveColStat.get(i));
- }
- return hiveColStatLst;
- } else {
- return m_hiveColStat;
- }
+ return ((RelOptHiveTable) table).getColStat(projIndxLst);
}
-
}
\ No newline at end of file
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushJoinThroughJoinRule.java Tue Jun 24 21:39:03 2014
@@ -27,7 +27,7 @@ public class HivePushJoinThroughJoinRule
final HiveJoinRel topJoin = call.rel(0);
final HiveJoinRel bottomJoin = call.rel(1);
- if (topJoin.getJoinAlgorithm() == JoinAlgorithm.NONE
+ if (!topJoin.isLeftSemiJoin() && topJoin.getJoinAlgorithm() == JoinAlgorithm.NONE
&& bottomJoin.getJoinAlgorithm() == JoinAlgorithm.NONE) {
isAMatch = true;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveSwapJoinRule.java Tue Jun 24 21:39:03 2014
@@ -16,7 +16,10 @@ public class HiveSwapJoinRule extends Sw
@Override
public boolean matches(RelOptRuleCall call) {
- return super.matches(call)
- && call.<HiveJoinRel> rel(0).getJoinAlgorithm() == JoinAlgorithm.NONE;
+ if (call.<HiveJoinRel> rel(0).isLeftSemiJoin())
+ return false;
+ else
+ return super.matches(call)
+ && call.<HiveJoinRel> rel(0).getJoinAlgorithm() == JoinAlgorithm.NONE;
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdDistinctRowCount.java Tue Jun 24 21:39:03 2014
@@ -6,6 +6,7 @@ import java.util.List;
import net.hydromatic.optiq.BuiltinMethod;
import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil;
+import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
import org.eigenbase.rel.JoinRelBase;
@@ -19,16 +20,18 @@ import org.eigenbase.rex.RexNode;
import org.eigenbase.util14.NumberUtil;
public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount {
- public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider.reflectiveSource(
- BuiltinMethod.DISTINCT_ROW_COUNT.method,
- new HiveRelMdDistinctRowCount());
+ public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider
+ .reflectiveSource(
+ BuiltinMethod.DISTINCT_ROW_COUNT.method,
+ new HiveRelMdDistinctRowCount());
private HiveRelMdDistinctRowCount() {
}
// Catch-all rule when none of the others apply.
@Override
- public Double getDistinctRowCount(RelNode rel, BitSet groupKey, RexNode predicate) {
+ public Double getDistinctRowCount(RelNode rel, BitSet groupKey,
+ RexNode predicate) {
if (rel instanceof HiveTableScanRel) {
return getDistinctRowCount((HiveTableScanRel) rel, groupKey, predicate);
}
@@ -37,8 +40,10 @@ public class HiveRelMdDistinctRowCount e
RelMetadataQuery.getSelectivity(rel, predicate));
}
- private Double getDistinctRowCount(HiveTableScanRel htRel, BitSet groupKey, RexNode predicate) {
- List<Integer> projIndxLst = HiveOptiqUtil.translateBitSetToProjIndx(groupKey);
+ private Double getDistinctRowCount(HiveTableScanRel htRel, BitSet groupKey,
+ RexNode predicate) {
+ List<Integer> projIndxLst = HiveOptiqUtil
+ .translateBitSetToProjIndx(groupKey);
List<ColStatistics> colStats = htRel.getColStat(projIndxLst);
Double noDistinctRows = 1.0;
for (ColStatistics cStat : colStats) {
@@ -51,12 +56,25 @@ public class HiveRelMdDistinctRowCount e
public static Double getDistinctRowCount(RelNode r, int indx) {
BitSet bitSetOfRqdProj = new BitSet();
bitSetOfRqdProj.set(indx);
- return RelMetadataQuery.getDistinctRowCount(r, bitSetOfRqdProj, r.getCluster().getRexBuilder()
- .makeLiteral(true));
+ return RelMetadataQuery.getDistinctRowCount(r, bitSetOfRqdProj, r
+ .getCluster().getRexBuilder().makeLiteral(true));
}
@Override
- public Double getDistinctRowCount(JoinRelBase rel, BitSet groupKey, RexNode predicate) {
- return RelMdUtil.getJoinDistinctRowCount(rel, rel.getJoinType(), groupKey, predicate, true);
+ public Double getDistinctRowCount(JoinRelBase rel, BitSet groupKey,
+ RexNode predicate) {
+ if (rel instanceof HiveJoinRel) {
+ HiveJoinRel hjRel = (HiveJoinRel) rel;
+ //TODO: Improve this
+ if (hjRel.isLeftSemiJoin()) {
+ return RelMetadataQuery.getDistinctRowCount(hjRel.getLeft(), groupKey,
+ rel.getCluster().getRexBuilder().makeLiteral(true));
+ } else {
+ return RelMdUtil.getJoinDistinctRowCount(rel, rel.getJoinType(),
+ groupKey, predicate, true);
+ }
+ }
+
+ return RelMetadataQuery.getDistinctRowCount(rel, groupKey, predicate);
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/HiveRelMdSelectivity.java Tue Jun 24 21:39:03 2014
@@ -50,15 +50,18 @@ public class HiveRelMdSelectivity extend
private Double computeInnerJoinSelectivity(HiveJoinRel j, RexNode predicate) {
double ndvCrossProduct = 1;
RexNode combinedPredicate = getCombinedPredicateForJoin(j, predicate);
- JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j, combinedPredicate);
- ImmutableMap.Builder<Integer, Double> colStatMapBuilder = ImmutableMap.builder();
+ JoinPredicateInfo jpi = JoinPredicateInfo.constructJoinPredicateInfo(j,
+ combinedPredicate);
+ ImmutableMap.Builder<Integer, Double> colStatMapBuilder = ImmutableMap
+ .builder();
ImmutableMap<Integer, Double> colStatMap;
int rightOffSet = j.getLeft().getRowType().getFieldCount();
// 1. Update Col Stats Map with col stats for columns from left side of
// Join which are part of join keys
for (Integer ljk : jpi.getProjsFromLeftPartOfJoinKeysInChildSchema()) {
- colStatMapBuilder.put(ljk, HiveRelMdDistinctRowCount.getDistinctRowCount(j.getLeft(), ljk));
+ colStatMapBuilder.put(ljk,
+ HiveRelMdDistinctRowCount.getDistinctRowCount(j.getLeft(), ljk));
}
// 2. Update Col Stats Map with col stats for columns from right side of
@@ -116,9 +119,12 @@ public class HiveRelMdSelectivity extend
ndvCrossProduct *= ndvToBeSmoothed;
}
- ndvCrossProduct = Math.min(
- RelMetadataQuery.getRowCount(j.getLeft()) * RelMetadataQuery.getRowCount(j.getRight()),
- ndvCrossProduct);
+ if (j.isLeftSemiJoin())
+ ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft()),
+ ndvCrossProduct);
+ else
+ ndvCrossProduct = Math.min(RelMetadataQuery.getRowCount(j.getLeft())
+ * RelMetadataQuery.getRowCount(j.getRight()), ndvCrossProduct);
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java Tue Jun 24 21:39:03 2014
@@ -41,21 +41,30 @@ class ASTBuilder {
return b.node();
}
- static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType, ASTNode cond) {
+ static ASTNode join(ASTNode left, ASTNode right, JoinRelType joinType,
+ ASTNode cond, boolean semiJoin) {
ASTBuilder b = null;
switch (joinType) {
case INNER:
- b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN");
+ if (semiJoin) {
+ b = ASTBuilder.construct(HiveParser.TOK_LEFTSEMIJOIN,
+ "TOK_LEFTSEMIJOIN");
+ } else {
+ b = ASTBuilder.construct(HiveParser.TOK_JOIN, "TOK_JOIN");
+ }
break;
case LEFT:
- b = ASTBuilder.construct(HiveParser.TOK_LEFTOUTERJOIN, "TOK_LEFTOUTERJOIN");
+ b = ASTBuilder.construct(HiveParser.TOK_LEFTOUTERJOIN,
+ "TOK_LEFTOUTERJOIN");
break;
case RIGHT:
- b = ASTBuilder.construct(HiveParser.TOK_RIGHTOUTERJOIN, "TOK_RIGHTOUTERJOIN");
+ b = ASTBuilder.construct(HiveParser.TOK_RIGHTOUTERJOIN,
+ "TOK_RIGHTOUTERJOIN");
break;
case FULL:
- b = ASTBuilder.construct(HiveParser.TOK_FULLOUTERJOIN, "TOK_FULLOUTERJOIN");
+ b = ASTBuilder.construct(HiveParser.TOK_FULLOUTERJOIN,
+ "TOK_FULLOUTERJOIN");
break;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java Tue Jun 24 21:39:03 2014
@@ -8,7 +8,9 @@ import java.util.concurrent.atomic.Atomi
import net.hydromatic.optiq.util.BitSets;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveJoinRel;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveSortRel;
+import org.apache.hadoop.hive.ql.optimizer.optiq.translator.SqlFunctionConverter.HiveToken;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
@@ -27,7 +29,6 @@ import org.eigenbase.rex.RexCall;
import org.eigenbase.rex.RexInputRef;
import org.eigenbase.rex.RexLiteral;
import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexUtil;
import org.eigenbase.rex.RexVisitorImpl;
import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.SqlOperator;
@@ -179,7 +180,10 @@ public class ASTConverter {
QueryBlockInfo right = convertSource(join.getRight());
s = new Schema(left.schema, right.schema);
ASTNode cond = join.getCondition().accept(new RexVisitor(s));
- ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond);
+ boolean semiJoin = ((join instanceof HiveJoinRel) && ((HiveJoinRel)join).isLeftSemiJoin()) ? true : false;
+ ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin);
+ if (semiJoin)
+ s = left.schema;
} else {
ASTConverter src = new ASTConverter(r);
ASTNode srcAST = src.convert(order);
@@ -266,9 +270,20 @@ public class ASTConverter {
SqlOperator op = call.getOperator();
List<ASTNode> astNodeLst = new LinkedList<ASTNode>();
+ if (op.kind == SqlKind.CAST) {
+ HiveToken ht = TypeConverter.convert(call.getType());
+ ASTBuilder astBldr = ASTBuilder.construct(ht.type, ht.text);
+ if (ht.args != null) {
+ for (String castArg : ht.args)
+ astBldr.add(HiveParser.Identifier, castArg);
+ }
+ astNodeLst.add(astBldr.node());
+ }
+
for (RexNode operand : call.operands) {
astNodeLst.add(operand.accept(this));
}
+
if (isFlat(call))
return SqlFunctionConverter.buildAST(op, astNodeLst, 0);
else
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RelNodeConverter.java Tue Jun 24 21:39:03 2014
@@ -73,12 +73,10 @@ import org.eigenbase.reltype.RelDataType
import org.eigenbase.rex.RexCall;
import org.eigenbase.rex.RexInputRef;
import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexUtil;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
import org.eigenbase.util.CompositeList;
import org.eigenbase.util.Pair;
-import com.esotericsoftware.minlog.Log;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
@@ -231,11 +229,11 @@ public class RelNodeConverter {
opPositionMap.put(node, opPositionMap.get(parent));
}
- RexNode convertToOptiqExpr(final ExprNodeDesc expr, final RelNode optiqOP, final boolean flatten) {
+ RexNode convertToOptiqExpr(final ExprNodeDesc expr, final RelNode optiqOP, final boolean flatten) throws SemanticException {
return convertToOptiqExpr(expr, optiqOP, 0, flatten);
}
- RexNode convertToOptiqExpr(final ExprNodeDesc expr, final RelNode optiqOP, int offset, final boolean flatten) {
+ RexNode convertToOptiqExpr(final ExprNodeDesc expr, final RelNode optiqOP, int offset, final boolean flatten) throws SemanticException {
ImmutableMap<String, Integer> posMap = opPositionMap.get(optiqOP);
RexNodeConverter c = new RexNodeConverter(cluster, optiqOP.getRowType(), posMap, offset, flatten);
return c.convert(expr);
@@ -268,7 +266,7 @@ public class RelNodeConverter {
* @todo: cleanup, for now just copied from HiveToOptiqRelConvereter
*/
private HiveJoinRel convertJoinOp(Context ctx, JoinOperator op, JoinCondDesc jc,
- HiveRel leftRel, HiveRel rightRel) {
+ HiveRel leftRel, HiveRel rightRel) throws SemanticException {
HiveJoinRel joinRel;
Operator<? extends OperatorDesc> leftParent = op.getParentOperators().get(jc.getLeft());
Operator<? extends OperatorDesc> rightParent = op.getParentOperators().get(jc.getRight());
@@ -350,17 +348,18 @@ public class RelNodeConverter {
}
}
- joinRel = HiveJoinRel.getJoin(ctx.cluster, leftRel, rightRel, joinPredicate, joinType);
+ joinRel = HiveJoinRel.getJoin(ctx.cluster, leftRel, rightRel, joinPredicate, joinType, false);
} else {
throw new RuntimeException("Right & Left of Join Condition columns are not equal");
}
return joinRel;
}
+
}
private static int convertExpr(Context ctx, RelNode input, ExprNodeDesc expr,
- List<RexNode> extraExprs) {
+ List<RexNode> extraExprs) throws SemanticException {
final RexNode rex = ctx.convertToOptiqExpr(expr, input, false);
final int index;
if (rex instanceof RexInputRef) {
@@ -373,7 +372,7 @@ public class RelNodeConverter {
}
private static AggregateCall convertAgg(Context ctx, AggregationDesc agg, RelNode input,
- ColumnInfo cI, List<RexNode> extraExprs) {
+ ColumnInfo cI, List<RexNode> extraExprs) throws SemanticException {
final Aggregation aggregation = AGG_MAP.get(agg.getGenericUDAFName());
if (aggregation == null) {
throw new AssertionError("agg not found: " + agg.getGenericUDAFName());
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java Tue Jun 24 21:39:03 2014
@@ -1,15 +1,35 @@
package org.apache.hadoop.hive.ql.optimizer.optiq.translator;
import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDecimal;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.eigenbase.rel.RelNode;
import org.eigenbase.relopt.RelOptCluster;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeFactory;
@@ -18,26 +38,48 @@ import org.eigenbase.rex.RexCall;
import org.eigenbase.rex.RexNode;
import org.eigenbase.sql.SqlOperator;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
public class RexNodeConverter {
- private final RelOptCluster m_cluster;
- private final RelDataType m_inpDataType;
- private final ImmutableMap<String, Integer> m_nameToPosMap;
- private final int m_offset;
- private final boolean m_flattenExpr;
+ private static class InputCtx {
+ private final RelDataType m_optiqInpDataType;
+ private final ImmutableMap<String, Integer> m_hiveNameToPosMap;
+ private final RowResolver m_hiveRR;
+ private final int m_offsetInOptiqSchema;
+
+ private InputCtx(RelDataType optiqInpDataType,
+ ImmutableMap<String, Integer> hiveNameToPosMap, RowResolver hiveRR,
+ int offsetInOptiqSchema) {
+ m_optiqInpDataType = optiqInpDataType;
+ m_hiveNameToPosMap = hiveNameToPosMap;
+ m_hiveRR = hiveRR;
+ m_offsetInOptiqSchema = offsetInOptiqSchema;
+ }
+ };
+
+ private final RelOptCluster m_cluster;
+ private final ImmutableList<InputCtx> m_inputCtxs;
+ private final boolean m_flattenExpr;
public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType,
- ImmutableMap<String, Integer> nameToPosMap, int offset, boolean flattenExpr) {
+ ImmutableMap<String, Integer> nameToPosMap, int offset,
+ boolean flattenExpr) {
+ this.m_cluster = cluster;
+ m_inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap,
+ null, offset));
+ m_flattenExpr = flattenExpr;
+ }
+
+ public RexNodeConverter(RelOptCluster cluster, List<InputCtx> inpCtxLst,
+ boolean flattenExpr) {
this.m_cluster = cluster;
- this.m_inpDataType = inpDataType;
- this.m_nameToPosMap = nameToPosMap;
- this.m_offset = offset;
+ m_inputCtxs = ImmutableList.<InputCtx> builder().addAll(inpCtxLst).build();
m_flattenExpr = flattenExpr;
}
- public RexNode convert(ExprNodeDesc expr) {
+ public RexNode convert(ExprNodeDesc expr) throws SemanticException {
if (expr instanceof ExprNodeGenericFuncDesc) {
return convert((ExprNodeGenericFuncDesc) expr);
} else if (expr instanceof ExprNodeConstantDesc) {
@@ -51,28 +93,109 @@ public class RexNodeConverter {
// ExprNodeColumnListDesc
}
- private RexNode convert(final ExprNodeGenericFuncDesc func) {
- SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getGenericUDF());
+ private RexNode convert(final ExprNodeGenericFuncDesc func)
+ throws SemanticException {
+
+ ExprNodeDesc tmpExprNode;
+ RexNode tmpRN;
+ TypeInfo tgtDT = null;
+
+ SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func
+ .getGenericUDF());
List<RexNode> childRexNodeLst = new LinkedList<RexNode>();
+ // TODO: 1) Expand to other functions as needed 2) What about types other
+ // than primitive
+ if (func.getGenericUDF() instanceof GenericUDFBaseNumeric) {
+ tgtDT = func.getTypeInfo();
+ } else if (func.getGenericUDF() instanceof GenericUDFBaseCompare) {
+ if (func.getChildren().size() == 2) {
+ tgtDT = FunctionRegistry.getCommonClassForComparison(func.getChildren()
+ .get(0).getTypeInfo(), func.getChildren().get(1).getTypeInfo());
+ }
+ }
+
for (ExprNodeDesc childExpr : func.getChildren()) {
- childRexNodeLst.add(convert(childExpr));
+ tmpExprNode = childExpr;
+ if (tgtDT != null
+ && TypeInfoUtils.isConversionRequiredForComparison(tgtDT,
+ childExpr.getTypeInfo())) {
+ tmpExprNode = ParseUtils.createConversionCast(childExpr,
+ (PrimitiveTypeInfo) tgtDT);
+ }
+ tmpRN = convert(tmpExprNode);
+ childRexNodeLst.add(tmpRN);
}
- RexNode convertedFilterExpr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
- if (m_flattenExpr && convertedFilterExpr instanceof RexCall) {
- RexCall call = (RexCall) convertedFilterExpr;
- convertedFilterExpr = m_cluster.getRexBuilder().makeFlatCall(call.getOperator(),
+ RexNode expr = null;
+
+ // This is an explicit cast
+ expr = handleExplicitCast(func, childRexNodeLst);
+
+ if (expr == null)
+ expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
+
+ if (m_flattenExpr && expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ expr = m_cluster.getRexBuilder().makeFlatCall(call.getOperator(),
call.getOperands());
}
- return convertedFilterExpr;
+ return expr;
}
- protected RexNode convert(ExprNodeColumnDesc col) {
- int pos = m_nameToPosMap.get(col.getColumn());
- return m_cluster.getRexBuilder().makeInputRef(m_inpDataType.getFieldList().get(pos).getType(),
- pos + m_offset);
+ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func,
+ List<RexNode> childRexNodeLst) {
+ RexNode castExpr = null;
+
+ if (childRexNodeLst != null && childRexNodeLst.size() == 1) {
+ GenericUDF udf = func.getGenericUDF();
+ if ((udf instanceof GenericUDFToChar)
+ || (udf instanceof GenericUDFToVarchar)
+ || (udf instanceof GenericUDFToDecimal)
+ || (udf instanceof GenericUDFToDate)
+ || (udf instanceof GenericUDFToBinary)
+ || (udf instanceof GenericUDFToUnixTimeStamp)) {
+ castExpr = m_cluster.getRexBuilder().makeCast(
+ TypeConverter.convert(func.getTypeInfo(),
+ m_cluster.getTypeFactory()), childRexNodeLst.get(0));
+ }
+ }
+
+ return castExpr;
+ }
+
+ private InputCtx getInputCtx(ExprNodeColumnDesc col) throws SemanticException {
+ InputCtx ctxLookingFor = null;
+
+ if (m_inputCtxs.size() == 1) {
+ ctxLookingFor = m_inputCtxs.get(0);
+ } else {
+ String tableAlias = col.getTabAlias();
+ String colAlias = col.getColumn();
+ int noInp = 0;
+ for (InputCtx ic : m_inputCtxs) {
+ if (tableAlias == null || ic.m_hiveRR.hasTableAlias(tableAlias)) {
+ if (ic.m_hiveRR.getPosition(colAlias) >= 0) {
+ ctxLookingFor = ic;
+ noInp++;
+ }
+ }
+ }
+
+ if (noInp > 1)
+ throw new RuntimeException("Ambigous column mapping");
+ }
+
+ return ctxLookingFor;
+ }
+
+ protected RexNode convert(ExprNodeColumnDesc col) throws SemanticException {
+ InputCtx ic = getInputCtx(col);
+ int pos = ic.m_hiveNameToPosMap.get(col.getColumn());
+ return m_cluster.getRexBuilder().makeInputRef(
+ ic.m_optiqInpDataType.getFieldList().get(pos).getType(),
+ pos + ic.m_offsetInOptiqSchema);
}
protected RexNode convert(ExprNodeConstantDesc literal) {
@@ -97,7 +220,8 @@ public class RexNodeConverter {
optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Short) value));
break;
case INT:
- optiqLiteral = rexBuilder.makeExactLiteral(new BigDecimal((Integer) value));
+ optiqLiteral = rexBuilder
+ .makeExactLiteral(new BigDecimal((Integer) value));
break;
case LONG:
optiqLiteral = rexBuilder.makeBigintLiteral(new BigDecimal((Long) value));
@@ -107,10 +231,12 @@ public class RexNodeConverter {
optiqLiteral = rexBuilder.makeExactLiteral((BigDecimal) value);
break;
case FLOAT:
- optiqLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Float) value), optiqDataType);
+ optiqLiteral = rexBuilder.makeApproxLiteral(
+ new BigDecimal((Float) value), optiqDataType);
break;
case DOUBLE:
- optiqLiteral = rexBuilder.makeApproxLiteral(new BigDecimal((Double) value), optiqDataType);
+ optiqLiteral = rexBuilder.makeApproxLiteral(
+ new BigDecimal((Double) value), optiqDataType);
break;
case STRING:
optiqLiteral = rexBuilder.makeLiteral((String) value);
@@ -127,4 +253,31 @@ public class RexNodeConverter {
return optiqLiteral;
}
+ public static RexNode getAlwaysTruePredicate(RelOptCluster cluster) {
+ SqlOperator optiqOp = SqlFunctionConverter
+ .getOptiqOperator(new GenericUDFOPEqual());
+ List<RexNode> childRexNodeLst = new LinkedList<RexNode>();
+ childRexNodeLst.add(cluster.getRexBuilder().makeLiteral(true));
+ childRexNodeLst.add(cluster.getRexBuilder().makeLiteral(true));
+
+ return cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
+ }
+
+ public static RexNode convert(RelOptCluster cluster,
+ ExprNodeDesc joinCondnExprNode, List<RelNode> inputRels,
+ LinkedHashMap<RelNode, RowResolver> relToHiveRR,
+ Map<RelNode, ImmutableMap<String, Integer>> relToHiveColNameOptiqPosMap,
+ boolean flattenExpr) throws SemanticException {
+ List<InputCtx> inputCtxLst = new ArrayList<InputCtx>();
+
+ int offSet = 0;
+ for (RelNode r : inputRels) {
+ inputCtxLst.add(new InputCtx(r.getRowType(), relToHiveColNameOptiqPosMap
+ .get(r), relToHiveRR.get(r), offSet));
+ offSet += r.getRowType().getFieldCount();
+ }
+
+ return (new RexNodeConverter(cluster, inputCtxLst, flattenExpr))
+ .convert(joinCondnExprNode);
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Tue Jun 24 21:39:03 2014
@@ -51,7 +51,9 @@ public class SqlFunctionConverter {
node = (ASTNode) ParseDriver.adaptor.create(hToken.type, hToken.text);
} else {
node = (ASTNode) ParseDriver.adaptor.create(HiveParser.TOK_FUNCTION, "TOK_FUNCTION");
- node.addChild((ASTNode) ParseDriver.adaptor.create(HiveParser.Identifier, op.getName()));
+ if (op.kind != SqlKind.CAST)
+ node.addChild((ASTNode) ParseDriver.adaptor.create(
+ HiveParser.Identifier, op.getName()));
}
for (ASTNode c : children) {
@@ -212,14 +214,18 @@ public class SqlFunctionConverter {
registerFunction(name, optiqFn, null);
}
- private void registerFunction(String name, SqlOperator optiqFn, HiveToken hiveToken) {
- FunctionInfo hFn = FunctionRegistry.getFunctionInfo(name);
+ private void registerFunction(String name, SqlOperator optiqFn,
+ HiveToken hiveToken) {
operatorMap.put(name, optiqFn);
- String hFnName = getName(hFn.getGenericUDF());
- hiveToOptiq.put(hFnName, optiqFn);
- if (hiveToken != null) {
- optiqToHiveToken.put(optiqFn, hiveToken);
+ FunctionInfo hFn = FunctionRegistry.getFunctionInfo(name);
+ if (hFn != null) {
+ String hFnName = getName(hFn.getGenericUDF());
+ hiveToOptiq.put(hFnName, optiqFn);
+
+ if (hiveToken != null) {
+ optiqToHiveToken.put(optiqFn, hiveToken);
+ }
}
}
}
@@ -228,17 +234,7 @@ public class SqlFunctionConverter {
return new HiveToken(type, text);
}
- static class HiveToken {
- int type;
- String text;
-
- HiveToken(int type, String text) {
- this.type = type;
- this.text = text;
- }
- }
-
- static SqlAggFunction hiveAggFunction(String name) {
+ public static SqlAggFunction hiveAggFunction(String name) {
return new HiveAggFunction(name);
}
@@ -258,4 +254,16 @@ public class SqlFunctionConverter {
}
}
+
+ static class HiveToken {
+ int type;
+ String text;
+ String[] args;
+
+ HiveToken(int type, String text, String... args) {
+ this.type = type;
+ this.text = text;
+ this.args = args;
+ }
+ }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java?rev=1605203&r1=1605202&r2=1605203&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java Tue Jun 24 21:39:03 2014
@@ -2,16 +2,25 @@ package org.apache.hadoop.hive.ql.optimi
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.optimizer.optiq.translator.SqlFunctionConverter.HiveToken;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.eigenbase.relopt.RelOptCluster;
import org.eigenbase.reltype.RelDataType;
@@ -20,11 +29,46 @@ import org.eigenbase.rex.RexBuilder;
import org.eigenbase.sql.type.SqlTypeName;
import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
import com.google.common.collect.Lists;
public class TypeConverter {
+ private static final Map<String, HiveToken> m_optiqToHiveTypeNameMap;
- public static RelDataType getType(RelOptCluster cluster, RowResolver rr, List<String> neededCols) {
+ // TODO: Handling of char[], varchar[], string...
+ static {
+ Builder<String, HiveToken> b = ImmutableMap.<String, HiveToken> builder();
+ b.put(SqlTypeName.BOOLEAN.getName(), new HiveToken(HiveParser.TOK_BOOLEAN, "TOK_BOOLEAN"));
+ b.put(SqlTypeName.TINYINT.getName(), new HiveToken(HiveParser.TOK_TINYINT, "TOK_TINYINT"));
+ b.put(SqlTypeName.SMALLINT.getName(), new HiveToken(HiveParser.TOK_SMALLINT, "TOK_SMALLINT"));
+ b.put(SqlTypeName.INTEGER.getName(), new HiveToken(HiveParser.TOK_INT, "TOK_INT"));
+ b.put(SqlTypeName.BIGINT.getName(), new HiveToken(HiveParser.TOK_BIGINT, "TOK_BIGINT"));
+ b.put(SqlTypeName.FLOAT.getName(), new HiveToken(HiveParser.TOK_FLOAT, "TOK_FLOAT"));
+ b.put(SqlTypeName.DOUBLE.getName(), new HiveToken(HiveParser.TOK_DOUBLE, "TOK_DOUBLE"));
+ b.put(SqlTypeName.DATE.getName(), new HiveToken(HiveParser.TOK_DATE, "TOK_DATE"));
+ b.put(SqlTypeName.TIMESTAMP.getName(), new HiveToken(HiveParser.TOK_TIMESTAMP, "TOK_TIMESTAMP"));
+ b.put(SqlTypeName.BINARY.getName(), new HiveToken(HiveParser.TOK_BINARY, "TOK_BINARY"));
+ m_optiqToHiveTypeNameMap = b.build();
+ };
+
+ /*********************** Convert Hive Types To Optiq Types ***********************/
+ public static RelDataType getType(RelOptCluster cluster,
+ List<ColumnInfo> cInfoLst) {
+ RexBuilder rexBuilder = cluster.getRexBuilder();
+ RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
+ List<RelDataType> fieldTypes = new LinkedList<RelDataType>();
+ List<String> fieldNames = new LinkedList<String>();
+
+ for (ColumnInfo ci : cInfoLst) {
+ fieldTypes.add(convert(ci.getType(), dtFactory));
+ fieldNames.add(ci.getInternalName());
+ }
+ return dtFactory.createStructType(fieldTypes, fieldNames);
+ }
+
+ public static RelDataType getType(RelOptCluster cluster, RowResolver rr,
+ List<String> neededCols) {
RexBuilder rexBuilder = cluster.getRexBuilder();
RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
RowSchema rs = rr.getRowSchema();
@@ -63,7 +107,8 @@ public class TypeConverter {
return convertedType;
}
- public static RelDataType convert(PrimitiveTypeInfo type, RelDataTypeFactory dtFactory) {
+ public static RelDataType convert(PrimitiveTypeInfo type,
+ RelDataTypeFactory dtFactory) {
RelDataType convertedType = null;
switch (type.getPrimitiveCategory()) {
@@ -105,7 +150,8 @@ public class TypeConverter {
convertedType = dtFactory.createSqlType(SqlTypeName.BINARY);
break;
case DECIMAL:
- convertedType = dtFactory.createSqlType(SqlTypeName.DECIMAL);
+ DecimalTypeInfo dtInf = (DecimalTypeInfo)type;
+ convertedType = dtFactory.createSqlType(SqlTypeName.DECIMAL, dtInf.precision(), dtInf.scale());
break;
case VARCHAR:
convertedType = dtFactory.createSqlType(SqlTypeName.VARCHAR,
@@ -123,30 +169,63 @@ public class TypeConverter {
return convertedType;
}
- public static RelDataType convert(ListTypeInfo lstType, RelDataTypeFactory dtFactory) {
+ public static RelDataType convert(ListTypeInfo lstType,
+ RelDataTypeFactory dtFactory) {
RelDataType elemType = convert(lstType.getListElementTypeInfo(), dtFactory);
return dtFactory.createArrayType(elemType, -1);
}
- public static RelDataType convert(MapTypeInfo mapType, RelDataTypeFactory dtFactory) {
+ public static RelDataType convert(MapTypeInfo mapType,
+ RelDataTypeFactory dtFactory) {
RelDataType keyType = convert(mapType.getMapKeyTypeInfo(), dtFactory);
RelDataType valueType = convert(mapType.getMapValueTypeInfo(), dtFactory);
return dtFactory.createMapType(keyType, valueType);
}
- public static RelDataType convert(StructTypeInfo structType, final RelDataTypeFactory dtFactory) {
- List<RelDataType> fTypes = Lists.transform(structType.getAllStructFieldTypeInfos(),
+ public static RelDataType convert(StructTypeInfo structType,
+ final RelDataTypeFactory dtFactory) {
+ List<RelDataType> fTypes = Lists.transform(
+ structType.getAllStructFieldTypeInfos(),
new Function<TypeInfo, RelDataType>() {
public RelDataType apply(TypeInfo tI) {
return convert(tI, dtFactory);
}
});
- return dtFactory.createStructType(fTypes, structType.getAllStructFieldNames());
+ return dtFactory.createStructType(fTypes,
+ structType.getAllStructFieldNames());
}
- public static RelDataType convert(UnionTypeInfo unionType, RelDataTypeFactory dtFactory) {
+ public static RelDataType convert(UnionTypeInfo unionType,
+ RelDataTypeFactory dtFactory) {
// @todo what do we about unions?
throw new UnsupportedOperationException();
}
+ /*********************** Convert Optiq Types To Hive Types ***********************/
+ public static HiveToken convert(RelDataType optiqType) {
+ HiveToken ht = null;
+
+ switch (optiqType.getSqlTypeName()) {
+ case CHAR: {
+ ht = new HiveToken(HiveParser.TOK_CHAR, "TOK_CHAR",
+ String.valueOf(optiqType.getPrecision()));
+ }
+ break;
+ case VARCHAR: {
+ ht = new HiveToken(HiveParser.TOK_VARCHAR, "TOK_VARCHAR",
+ String.valueOf(optiqType.getPrecision()));
+ }
+ break;
+ case DECIMAL: {
+ ht = new HiveToken(HiveParser.TOK_DECIMAL, "TOK_DECIMAL",
+ String.valueOf(optiqType.getPrecision()), String.valueOf(optiqType
+ .getScale()));
+ }
+ break;
+ default:
+ ht = m_optiqToHiveTypeNameMap.get(optiqType.getSqlTypeName().getName());
+ }
+
+ return ht;
+ }
}