You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/11/08 21:25:40 UTC
svn commit: r1637593 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/tez/ optimizer/optiq/rules/ optimizer/optiq/stats/
optimizer/optiq/translator/
Author: gunther
Date: Sat Nov 8 20:25:39 2014
New Revision: 1637593
URL: http://svn.apache.org/r1637593
Log:
Revert HIVE-8768: Committed 2 extra files by mistake (Gunther Hagleitner)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1637593&r1=1637592&r2=1637593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java Sat Nov 8 20:25:39 2014
@@ -110,13 +110,9 @@ public class TezJobMonitor {
/* Pretty print the values */
private final NumberFormat secondsFormat;
private final NumberFormat commaFormat;
- private static List<DAGClient> shutdownList;
-
- public static void initShutdownHook() {
- if (shutdownList != null) {
- return;
- }
+ private static final List<DAGClient> shutdownList;
+ static {
shutdownList = Collections.synchronizedList(new LinkedList<DAGClient>());
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
@@ -134,10 +130,9 @@ public class TezJobMonitor {
}
}
});
- }
+ }
public TezJobMonitor() {
- initShutdownHook();
console = SessionState.getConsole();
secondsFormat = new DecimalFormat("#0.00");
commaFormat = NumberFormat.getNumberInstance(Locale.US);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1637593&r1=1637592&r2=1637593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java Sat Nov 8 20:25:39 2014
@@ -187,7 +187,6 @@ public class TezSessionState {
LOG.info("Opening new Tez Session (id: " + sessionId
+ ", scratch dir: " + tezScratchDir + ")");
- TezJobMonitor.initShutdownHook();
session.start();
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_PREWARM_ENABLED)) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java?rev=1637593&r1=1637592&r2=1637593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HivePushFilterPastJoinRule.java Sat Nov 8 20:25:39 2014
@@ -95,40 +95,36 @@ public abstract class HivePushFilterPast
* not equi join conditions.
*/
@Override
- protected void validateJoinFilters(List<RexNode> aboveFilters, List<RexNode> joinFilters,
- JoinRelBase join, JoinRelType joinType) {
- if (joinType.equals(JoinRelType.INNER)) {
- ListIterator<RexNode> filterIter = joinFilters.listIterator();
- while (filterIter.hasNext()) {
- RexNode exp = filterIter.next();
- if (exp instanceof RexCall) {
- RexCall c = (RexCall) exp;
- if ((c.getOperator().getKind() == SqlKind.EQUALS)
- || (c.getOperator().getKind() == SqlKind.LESS_THAN)
- || (c.getOperator().getKind() == SqlKind.GREATER_THAN)
- || (c.getOperator().getKind() == SqlKind.LESS_THAN_OR_EQUAL)
- || (c.getOperator().getKind() == SqlKind.GREATER_THAN_OR_EQUAL)) {
- boolean validHiveJoinFilter = true;
- for (RexNode rn : c.getOperands()) {
- // NOTE: Hive dis-allows projections from both left
- // &
- // right side
- // of join condition. Example: Hive disallows
- // (r1.x=r2.x)=(r1.y=r2.y) on join condition.
- if (filterRefersToBothSidesOfJoin(rn, join)) {
- validHiveJoinFilter = false;
- break;
- }
- }
- if (validHiveJoinFilter)
- continue;
- }
- }
- aboveFilters.add(exp);
- filterIter.remove();
- }
- }
- }
+ protected void validateJoinFilters(List<RexNode> aboveFilters,
+ List<RexNode> joinFilters, JoinRelBase join, JoinRelType joinType) {
+ if (joinType.equals(JoinRelType.INNER)) {
+ ListIterator<RexNode> filterIter = joinFilters.listIterator();
+ while (filterIter.hasNext()) {
+ RexNode exp = filterIter.next();
+ if (exp instanceof RexCall) {
+ RexCall c = (RexCall) exp;
+ if (c.getOperator().getKind() == SqlKind.EQUALS) {
+ boolean validHiveJoinFilter = true;
+ for (RexNode rn : c.getOperands()) {
+ // NOTE: Hive dis-allows projections from both left
+ // &
+ // right side
+ // of join condition. Example: Hive disallows
+ // (r1.x=r2.x)=(r1.y=r2.y) on join condition.
+ if (filterRefersToBothSidesOfJoin(rn, join)) {
+ validHiveJoinFilter = false;
+ break;
+ }
+ }
+ if (validHiveJoinFilter)
+ continue;
+ }
+ }
+ aboveFilters.add(exp);
+ filterIter.remove();
+ }
+ }
+ }
private boolean filterRefersToBothSidesOfJoin(RexNode filter, JoinRelBase j) {
boolean refersToBothSides = false;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java?rev=1637593&r1=1637592&r2=1637593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/stats/FilterSelectivityEstimator.java Sat Nov 8 20:25:39 2014
@@ -18,9 +18,6 @@
package org.apache.hadoop.hive.ql.optimizer.optiq.stats;
import java.util.BitSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel;
@@ -35,10 +32,6 @@ import org.eigenbase.rex.RexInputRef;
import org.eigenbase.rex.RexNode;
import org.eigenbase.rex.RexVisitorImpl;
import org.eigenbase.sql.SqlKind;
-import org.eigenbase.sql.SqlOperator;
-import org.eigenbase.sql.type.SqlTypeUtil;
-
-import com.google.common.collect.Sets;
public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
private final RelNode childRel;
@@ -68,7 +61,7 @@ public class FilterSelectivityEstimator
}
Double selectivity = null;
- SqlKind op = getOp(call);
+ SqlKind op = call.getKind();
switch (op) {
case AND: {
@@ -81,7 +74,6 @@ public class FilterSelectivityEstimator
break;
}
- case NOT:
case NOT_EQUALS: {
selectivity = computeNotEqualitySelectivity(call);
break;
@@ -96,16 +88,7 @@ public class FilterSelectivityEstimator
}
case IN: {
- // TODO: 1) check for duplicates 2) We assume in clause values to be
- // present in NDV which may not be correct (Range check can find it) 3) We
- // assume values in NDV set is uniformly distributed over col values
- // (account for skewness - histogram).
- selectivity = computeFunctionSelectivity(call) * (call.operands.size() - 1);
- if (selectivity <= 0.0) {
- selectivity = 0.10;
- } else if (selectivity >= 1.0) {
- selectivity = 1.0;
- }
+ selectivity = ((double) 1 / ((double) call.operands.size()));
break;
}
@@ -169,19 +152,18 @@ public class FilterSelectivityEstimator
}
tmpCardinality = childCardinality * tmpSelectivity;
- if (tmpCardinality > 1 && tmpCardinality < childCardinality) {
+ if (tmpCardinality > 1)
tmpSelectivity = (1 - tmpCardinality / childCardinality);
- } else {
+ else
tmpSelectivity = 1.0;
- }
selectivity *= tmpSelectivity;
}
- if (selectivity < 0.0)
- selectivity = 0.0;
-
- return (1 - selectivity);
+ if (selectivity > 1)
+ return (1 - selectivity);
+ else
+ return 1.0;
}
/**
@@ -243,19 +225,4 @@ public class FilterSelectivityEstimator
}
return false;
}
-
- private SqlKind getOp(RexCall call) {
- SqlKind op = call.getKind();
-
- if (call.getKind().equals(SqlKind.OTHER_FUNCTION)
- && SqlTypeUtil.inBooleanFamily(call.getType())) {
- SqlOperator sqlOp = call.getOperator();
- String opName = (sqlOp != null) ? sqlOp.getName() : "";
- if (opName.equalsIgnoreCase("in")) {
- op = SqlKind.IN;
- }
- }
-
- return op;
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1637593&r1=1637592&r2=1637593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Sat Nov 8 20:25:39 2014
@@ -278,7 +278,6 @@ public class SqlFunctionConverter {
registerFunction(">=", SqlStdOperatorTable.GREATER_THAN_OR_EQUAL,
hToken(HiveParser.GREATERTHANOREQUALTO, ">="));
registerFunction("!", SqlStdOperatorTable.NOT, hToken(HiveParser.KW_NOT, "not"));
- registerFunction("<>", SqlStdOperatorTable.NOT_EQUALS, hToken(HiveParser.NOTEQUAL, "<>"));
}
private void registerFunction(String name, SqlOperator optiqFn, HiveToken hiveToken) {