You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/10/17 17:33:57 UTC
svn commit: r1632609 - in /hive/branches/branch-0.14/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/
java/org/apache/hadoop/hive/ql/optimizer/pcr/
java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/
test/results/clientpositive/
Author: hashutosh
Date: Fri Oct 17 15:33:57 2014
New Revision: 1632609
URL: http://svn.apache.org/r1632609
Log:
HIVE-8428 : PCR doesnt remove filters involving casts (Ashutosh Chauhan via Sergey Shelukhin)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_ppd_decimal.q
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/pcr.q
hive/branches/branch-0.14/ql/src/test/results/clientpositive/pcr.q.out
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java Fri Oct 17 15:33:57 2014
@@ -187,6 +187,14 @@ public final class ConstantPropagateProc
return new ExprNodeConstantDesc(ti, convObj);
}
+ public static ExprNodeDesc foldExpr(ExprNodeGenericFuncDesc funcDesc) {
+
+ GenericUDF udf = funcDesc.getGenericUDF();
+ if (!isDeterministicUdf(udf)) {
+ return funcDesc;
+ }
+ return evaluateFunction(funcDesc.getGenericUDF(),funcDesc.getChildren(), funcDesc.getChildren());
+ }
/**
* Fold input expression desc.
*
@@ -278,7 +286,7 @@ public final class ConstantPropagateProc
(UDF) Class.forName(bridge.getUdfClassName(), true, Utilities.getSessionSpecifiedClassLoader())
.newInstance();
files = udfInternal.getRequiredFiles();
- jars = udf.getRequiredJars();
+ jars = udfInternal.getRequiredJars();
} catch (Exception e) {
LOG.error("The UDF implementation class '" + udfClassName
+ "' is not present in the class path");
@@ -470,6 +478,16 @@ public final class ConstantPropagateProc
// FIXME: add null support.
return null;
+ } else if (desc instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc)desc);
+ if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) {
+ return null;
+ }
+ ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn;
+ Object writableValue = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
+ (PrimitiveTypeInfo) constant.getTypeInfo()).getPrimitiveWritableObject(constant.getValue());
+ arguments[i] = new DeferredJavaObject(writableValue);
+ argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue);
} else {
return null;
}
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrOpProcFactory.java Fri Oct 17 15:33:57 2014
@@ -31,10 +31,13 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
/**
@@ -133,7 +136,15 @@ public final class PcrOpProcFactory {
if (wrapper.state == PcrExprProcFactory.WalkState.TRUE) {
owc.getOpToRemove().add(new PcrOpWalkerCtx.OpToDeleteInfo(pop, fop));
- } else if (wrapper.state != PcrExprProcFactory.WalkState.FALSE) {
+ } else if (wrapper.state == PcrExprProcFactory.WalkState.CONSTANT && wrapper.outExpr instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeDesc desc = ConstantPropagateProcFactory.foldExpr((ExprNodeGenericFuncDesc)wrapper.outExpr);
+ if (desc != null && desc instanceof ExprNodeConstantDesc && Boolean.TRUE.equals(((ExprNodeConstantDesc)desc).getValue())) {
+ owc.getOpToRemove().add(new PcrOpWalkerCtx.OpToDeleteInfo(pop, fop));
+ } else {
+ fop.getConf().setPredicate(wrapper.outExpr);
+ }
+ }
+ else if (wrapper.state != PcrExprProcFactory.WalkState.FALSE) {
fop.getConf().setPredicate(wrapper.outExpr);
} else {
LOG.warn("Filter passes no row");
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Fri Oct 17 15:33:57 2014
@@ -306,7 +306,9 @@ public class TezCompiler extends TaskCom
// need a new run of the constant folding because we might have created lots
// of "and true and true" conditions.
- new ConstantPropagate().transform(procCtx.parseContext);
+ if(procCtx.conf.getBoolVar(ConfVars.HIVEOPTCONSTANTPROPAGATION)) {
+ new ConstantPropagate().transform(procCtx.parseContext);
+ }
}
@Override
Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_ppd_decimal.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_ppd_decimal.q?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_ppd_decimal.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_ppd_decimal.q Fri Oct 17 15:33:57 2014
@@ -22,7 +22,9 @@ set hive.optimize.index.filter=false;
select sum(hash(*)) from newtypesorc where d=cast('0.22' as float);
set hive.optimize.index.filter=true;
+set hive.optimize.constant.propagation=false;
select sum(hash(*)) from newtypesorc where d=cast('0.22' as float);
+set hive.optimize.constant.propagation=true;
set hive.optimize.index.filter=false;
select sum(hash(*)) from newtypesorc where d!=0.22;
Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/orc_vectorization_ppd.q Fri Oct 17 15:33:57 2014
@@ -13,12 +13,14 @@ stored as ORC tblproperties("orc.row.ind
-- insert creates separate orc files
insert overwrite table vectororc select "apple", "a", rand(1), "zoo" from srcorc;
+set hive.optimize.constant.propagation=false;
insert into table vectororc select null, "b", rand(2), "zoo" from srcorc;
insert into table vectororc select null, "c", rand(3), "zoo" from srcorc;
insert into table vectororc select "apple", "d", rand(4), "zoo" from srcorc;
insert into table vectororc select null, "e", rand(5), "z" from srcorc;
insert into table vectororc select "apple", "f", rand(6), "z" from srcorc;
insert into table vectororc select null, "g", rand(7), "zoo" from srcorc;
+set hive.optimize.constant.propagation=true;
-- since vectororc table has multiple orc file we will load them into a single file using another table
create table if not exists testorc
Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/pcr.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/pcr.q?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/pcr.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/pcr.q Fri Oct 17 15:33:57 2014
@@ -138,4 +138,6 @@ insert overwrite table foo_field partiti
select s,ds from foo_field where ((ds + s.a) > 0) order by ds,s;
drop table foo_field;
-
+explain select key,value from srcpart where cast(hr as double) = cast(11 as double);
+explain select key,value from srcpart where hr = cast(11 as double);
+explain select key,value from srcpart where cast(hr as double) = 11 ;
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/pcr.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/pcr.q.out?rev=1632609&r1=1632608&r2=1632609&view=diff
==============================================================================
Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/pcr.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/pcr.q.out Fri Oct 17 15:33:57 2014 differ