You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/11/27 21:11:46 UTC
svn commit: r1546180 [1/2] - in /hive/branches/tez: ./
common/src/java/org/apache/hadoop/hive/conf/
common/src/test/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ ql/src/java/org/...
Author: gunther
Date: Wed Nov 27 20:11:44 2013
New Revision: 1546180
URL: http://svn.apache.org/r1546180
Log:
Merge latest trunk into branch. (Gunther Hagleitner)
Added:
hive/branches/tez/common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java
- copied unchanged from r1546164, hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConfRestrictList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContextRegion.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContextRegion.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseUnary.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseUnary.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCeil.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCeil.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloor.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloorCeilBase.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloorCeilBase.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNegative.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNegative.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPositive.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPositive.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java
- copied unchanged from r1546164, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFAbs.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFAbs.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFCeil.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFCeil.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDecode.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDecode.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFEncode.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFEncode.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFloor.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFloor.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNegative.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNegative.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPositive.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPPositive.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPower.java
- copied unchanged from r1546164, hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPower.java
hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_context.q
- copied unchanged from r1546164, hive/trunk/ql/src/test/queries/clientpositive/vectorized_context.q
hive/branches/tez/ql/src/test/results/clientpositive/vectorized_context.q.out
- copied unchanged from r1546164, hive/trunk/ql/src/test/results/clientpositive/vectorized_context.q.out
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testShutdownBeforeExec.approved.txt
- copied unchanged from r1546164, hive/trunk/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.testShutdownBeforeExec.approved.txt
Removed:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JobCloseFeedBack.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFDecode.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBaseNumericUnaryOp.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCeil.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFFloor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPNegative.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFOPPositive.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFAbs.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDecode.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFEncode.java
Modified:
hive/branches/tez/ (props changed)
hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
hive/branches/tez/ql/src/test/queries/clientpositive/orc_split_elimination.q
hive/branches/tez/ql/src/test/results/clientpositive/decimal_udf.q.out
hive/branches/tez/ql/src/test/results/clientpositive/literal_decimal.q.out
hive/branches/tez/ql/src/test/results/clientpositive/orc_split_elimination.q.out
hive/branches/tez/ql/src/test/results/clientpositive/udf4.q.out
hive/branches/tez/ql/src/test/results/clientpositive/udf7.q.out
hive/branches/tez/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
hive/branches/tez/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out
hive/branches/tez/ql/src/test/results/compiler/plan/udf4.q.xml
hive/branches/tez/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PTest.java
hive/branches/tez/testutils/ptest2/src/main/resources/batch-exec.vm
hive/branches/tez/testutils/ptest2/src/main/resources/source-prep.vm
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestHostExecutor.java
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.java
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testAlternativeTestJVM.approved.txt
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepGit.approved.txt
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepNone.approved.txt
hive/branches/tez/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testPrepSvn.approved.txt
Propchange: hive/branches/tez/
------------------------------------------------------------------------------
Merged /hive/trunk:r1545526-1546164
Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Nov 27 20:11:44 2013
@@ -1240,14 +1240,7 @@ public class HiveConf extends Configurat
}
// setup list of conf vars that are not allowed to change runtime
- String restrictListStr = this.get(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString(), "").trim();
- for (String entry : restrictListStr.split(",")) {
- entry = entry.trim();
- if (!entry.isEmpty()) {
- restrictList.add(entry);
- }
- }
- restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString());
+ setupRestrictList();
}
@@ -1420,4 +1413,36 @@ public class HiveConf extends Configurat
return null;
}
}
+
+ /**
+ * Append comma separated list of config vars to the restrict List
+ * @param restrictListStr
+ */
+ public void addToRestrictList(String restrictListStr) {
+ if (restrictListStr == null) {
+ return;
+ }
+ String oldList = this.getVar(ConfVars.HIVE_CONF_RESTRICTED_LIST);
+ if (oldList == null || oldList.isEmpty()) {
+ this.setVar(ConfVars.HIVE_CONF_RESTRICTED_LIST, restrictListStr);
+ } else {
+ this.setVar(ConfVars.HIVE_CONF_RESTRICTED_LIST, oldList + "," + restrictListStr);
+ }
+ setupRestrictList();
+ }
+
+ /**
+ * Add the HIVE_CONF_RESTRICTED_LIST values to restrictList,
+ * including HIVE_CONF_RESTRICTED_LIST itself
+ */
+ private void setupRestrictList() {
+ String restrictListStr = this.getVar(ConfVars.HIVE_CONF_RESTRICTED_LIST);
+ restrictList.clear();
+ if (restrictListStr != null) {
+ for (String entry : restrictListStr.split(",")) {
+ restrictList.add(entry.trim());
+ }
+ }
+ restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.toString());
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Wed Nov 27 20:11:44 2013
@@ -878,7 +878,7 @@ public class FileSinkOperator extends Te
}
@Override
- public void jobCloseOp(Configuration hconf, boolean success, JobCloseFeedBack feedBack)
+ public void jobCloseOp(Configuration hconf, boolean success)
throws HiveException {
try {
if ((conf != null) && isNativeTable) {
@@ -893,7 +893,7 @@ public class FileSinkOperator extends Te
} catch (IOException e) {
throw new HiveException(e);
}
- super.jobCloseOp(hconf, success, feedBack);
+ super.jobCloseOp(hconf, success);
}
@Override
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Wed Nov 27 20:11:44 2013
@@ -44,8 +44,6 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.udf.GenericUDFDecode;
-import org.apache.hadoop.hive.ql.udf.GenericUDFEncode;
import org.apache.hadoop.hive.ql.udf.SettableUDF;
import org.apache.hadoop.hive.ql.udf.UDAFPercentile;
import org.apache.hadoop.hive.ql.udf.UDFAcos;
@@ -54,7 +52,6 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFAtan;
import org.apache.hadoop.hive.ql.udf.UDFBase64;
import org.apache.hadoop.hive.ql.udf.UDFBin;
-import org.apache.hadoop.hive.ql.udf.UDFCeil;
import org.apache.hadoop.hive.ql.udf.UDFConv;
import org.apache.hadoop.hive.ql.udf.UDFCos;
import org.apache.hadoop.hive.ql.udf.UDFDate;
@@ -66,7 +63,6 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFE;
import org.apache.hadoop.hive.ql.udf.UDFExp;
import org.apache.hadoop.hive.ql.udf.UDFFindInSet;
-import org.apache.hadoop.hive.ql.udf.UDFFloor;
import org.apache.hadoop.hive.ql.udf.UDFFromUnixTime;
import org.apache.hadoop.hive.ql.udf.UDFHex;
import org.apache.hadoop.hive.ql.udf.UDFHour;
@@ -86,11 +82,8 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFOPBitOr;
import org.apache.hadoop.hive.ql.udf.UDFOPBitXor;
import org.apache.hadoop.hive.ql.udf.UDFOPLongDivide;
-import org.apache.hadoop.hive.ql.udf.UDFOPNegative;
-import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
import org.apache.hadoop.hive.ql.udf.UDFPI;
import org.apache.hadoop.hive.ql.udf.UDFParseUrl;
-import org.apache.hadoop.hive.ql.udf.UDFPower;
import org.apache.hadoop.hive.ql.udf.UDFRTrim;
import org.apache.hadoop.hive.ql.udf.UDFRadians;
import org.apache.hadoop.hive.ql.udf.UDFRand;
@@ -197,10 +190,10 @@ public final class FunctionRegistry {
registerGenericUDF("size", GenericUDFSize.class);
registerGenericUDF("round", GenericUDFRound.class);
- registerUDF("floor", UDFFloor.class, false);
+ registerGenericUDF("floor", GenericUDFFloor.class);
registerUDF("sqrt", UDFSqrt.class, false);
- registerUDF("ceil", UDFCeil.class, false);
- registerUDF("ceiling", UDFCeil.class, false);
+ registerGenericUDF("ceil", GenericUDFCeil.class);
+ registerGenericUDF("ceiling", GenericUDFCeil.class);
registerUDF("rand", UDFRand.class, false);
registerGenericUDF("abs", GenericUDFAbs.class);
registerGenericUDF("pmod", GenericUDFPosMod.class);
@@ -214,8 +207,8 @@ public final class FunctionRegistry {
registerUDF("log10", UDFLog10.class, false);
registerUDF("log", UDFLog.class, false);
registerUDF("exp", UDFExp.class, false);
- registerUDF("power", UDFPower.class, false);
- registerUDF("pow", UDFPower.class, false);
+ registerGenericUDF("power", GenericUDFPower.class);
+ registerGenericUDF("pow", GenericUDFPower.class);
registerUDF("sign", UDFSign.class, false);
registerUDF("pi", UDFPI.class, false);
registerUDF("degrees", UDFDegrees.class, false);
@@ -257,8 +250,8 @@ public final class FunctionRegistry {
registerGenericUDF("str_to_map", GenericUDFStringToMap.class);
registerGenericUDF("translate", GenericUDFTranslate.class);
- registerUDF("positive", UDFOPPositive.class, true, "+");
- registerUDF("negative", UDFOPNegative.class, true, "-");
+ registerGenericUDF("positive", GenericUDFOPPositive.class);
+ registerGenericUDF("negative", GenericUDFOPNegative.class);
registerUDF("day", UDFDayOfMonth.class, false);
registerUDF("dayofmonth", UDFDayOfMonth.class, false);
@@ -1430,17 +1423,12 @@ public final class FunctionRegistry {
* Get the UDF class from an exprNodeDesc. Returns null if the exprNodeDesc
* does not contain a UDF class.
*/
- private static Class<? extends UDF> getUDFClassFromExprDesc(ExprNodeDesc desc) {
+ private static Class<? extends GenericUDF> getUDFClassFromExprDesc(ExprNodeDesc desc) {
if (!(desc instanceof ExprNodeGenericFuncDesc)) {
return null;
}
ExprNodeGenericFuncDesc genericFuncDesc = (ExprNodeGenericFuncDesc) desc;
- if (!(genericFuncDesc.getGenericUDF() instanceof GenericUDFBridge)) {
- return null;
- }
- GenericUDFBridge bridge = (GenericUDFBridge) (genericFuncDesc
- .getGenericUDF());
- return bridge.getUdfClass();
+ return genericFuncDesc.getGenericUDF().getClass();
}
/**
@@ -1533,8 +1521,8 @@ public final class FunctionRegistry {
* Returns whether the exprNodeDesc is a node of "positive".
*/
public static boolean isOpPositive(ExprNodeDesc desc) {
- Class<? extends UDF> udfClass = getUDFClassFromExprDesc(desc);
- return UDFOPPositive.class == udfClass;
+ Class<? extends GenericUDF> udfClass = getUDFClassFromExprDesc(desc);
+ return GenericUDFOPPositive.class == udfClass;
}
/**
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Wed Nov 27 20:11:44 2013
@@ -144,7 +144,7 @@ public class JoinOperator extends Common
}
@Override
- public void jobCloseOp(Configuration hconf, boolean success, JobCloseFeedBack feedBack)
+ public void jobCloseOp(Configuration hconf, boolean success)
throws HiveException {
int numAliases = conf.getExprs().size();
if (conf.getHandleSkewJoin()) {
@@ -181,7 +181,7 @@ public class JoinOperator extends Common
throw new HiveException(e);
}
}
- super.jobCloseOp(hconf, success, feedBack);
+ super.jobCloseOp(hconf, success);
}
private void moveUpFiles(String specPath, Configuration hconf, Log log)
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Wed Nov 27 20:11:44 2013
@@ -598,7 +598,7 @@ public abstract class Operator<T extends
private boolean jobCloseDone = false;
// Operator specific logic goes here
- public void jobCloseOp(Configuration conf, boolean success, JobCloseFeedBack feedBack)
+ public void jobCloseOp(Configuration conf, boolean success)
throws HiveException {
}
@@ -611,19 +611,19 @@ public abstract class Operator<T extends
* @param success
* whether the job was completed successfully or not
*/
- public void jobClose(Configuration conf, boolean success, JobCloseFeedBack feedBack)
+ public void jobClose(Configuration conf, boolean success)
throws HiveException {
// JobClose has already been performed on this operator
if (jobCloseDone) {
return;
}
- jobCloseOp(conf, success, feedBack);
+ jobCloseOp(conf, success);
jobCloseDone = true;
if (childOperators != null) {
for (Operator<? extends OperatorDesc> op : childOperators) {
- op.jobClose(conf, success, feedBack);
+ op.jobClose(conf, success);
}
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Wed Nov 27 20:11:44 2013
@@ -50,7 +50,6 @@ import org.apache.hadoop.hive.ql.ErrorMs
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.exec.FetchOperator;
import org.apache.hadoop.hive.ql.exec.HiveTotalOrderPartitioner;
-import org.apache.hadoop.hive.ql.exec.JobCloseFeedBack;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorUtils;
import org.apache.hadoop.hive.ql.exec.PartitionKeySampler;
@@ -466,14 +465,13 @@ public class ExecDriver extends Task<Map
// get the list of Dynamic partition paths
try {
if (rj != null) {
- JobCloseFeedBack feedBack = new JobCloseFeedBack();
if (mWork.getAliasToWork() != null) {
for (Operator<? extends OperatorDesc> op : mWork.getAliasToWork().values()) {
- op.jobClose(job, success, feedBack);
+ op.jobClose(job, success);
}
}
if (rWork != null) {
- rWork.getReducer().jobClose(job, success, feedBack);
+ rWork.getReducer().jobClose(job, success);
}
}
} catch (Exception e) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Wed Nov 27 20:11:44 2013
@@ -35,7 +35,6 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.exec.JobCloseFeedBack;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -256,12 +255,11 @@ public class TezTask extends Task<TezWor
*/
private int close(TezWork work, int rc) {
try {
- JobCloseFeedBack feedBack = new JobCloseFeedBack();
List<BaseWork> ws = work.getAllWork();
for (BaseWork w: ws) {
List<Operator<?>> ops = w.getAllOperators();
for (Operator<?> op: ops) {
- op.jobClose(conf, rc == 0, feedBack);
+ op.jobClose(conf, rc == 0);
}
}
} catch (Exception e) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java Wed Nov 27 20:11:44 2013
@@ -28,6 +28,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
@@ -39,11 +40,12 @@ import org.apache.hadoop.hive.ql.plan.Ma
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
/**
* The vectorized version of the MapJoinOperator.
*/
-public class VectorMapJoinOperator extends MapJoinOperator {
+public class VectorMapJoinOperator extends MapJoinOperator implements VectorizationContextRegion {
private static final Log LOG = LogFactory.getLog(
VectorMapJoinOperator.class.getName());
@@ -77,7 +79,10 @@ public class VectorMapJoinOperator exten
//
private transient int batchIndex;
private transient VectorHashKeyWrapper[] keyValues;
-
+
+ private transient VectorizationContext vOutContext = null;
+ private transient VectorizedRowBatchCtx vrbCtx = null;
+
public VectorMapJoinOperator() {
super();
}
@@ -113,36 +118,28 @@ public class VectorMapJoinOperator exten
bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable));
List<String> outColNames = desc.getOutputColumnNames();
- int outputColumnIndex = 0;
-
- Map<String, Integer> cMap = vContext.getColumnMap();
- for(byte alias:order) {
- for(ExprNodeDesc expr: exprs.get(alias)) {
- String columnName = outColNames.get(outputColumnIndex);
- if (!cMap.containsKey(columnName)) {
- vContext.addOutputColumn(columnName, expr.getTypeString());
- }
- ++outputColumnIndex;
- }
+
+ Map<String, Integer> mapOutCols = new HashMap<String, Integer>(outColNames.size());
+
+ int outColIndex = 0;
+ for(String outCol: outColNames) {
+ mapOutCols.put(outCol, outColIndex++);
}
-
- this.fileKey = vContext.getFileKey();
+
+ vOutContext = new VectorizationContext(mapOutCols, outColIndex);
+ vOutContext.setFileKey(vContext.getFileKey() + "/MAP_JOIN_" + desc.getBigTableAlias());
+ this.fileKey = vOutContext.getFileKey();
}
@Override
public void initializeOp(Configuration hconf) throws HiveException {
super.initializeOp(hconf);
+
- Map<String, Map<Integer, String>> allTypeMaps = Utilities.
- getMapRedWork(hconf).getMapWork().getScratchColumnVectorTypes();
- Map<Integer, String> typeMap = allTypeMaps.get(fileKey);
-
- Map<String, Map<String, Integer>> allColumnMaps = Utilities.
- getMapRedWork(hconf).getMapWork().getScratchColumnMap();
-
- Map<String, Integer> columnMap = allColumnMaps.get(fileKey);
-
- outputBatch = VectorizedRowBatch.buildBatch(typeMap, columnMap);
+ vrbCtx = new VectorizedRowBatchCtx();
+ vrbCtx.init(hconf, this.fileKey, (StructObjectInspector) this.outputObjInspector);
+
+ outputBatch = vrbCtx.createVectorizedRowBatch();
keyWrapperBatch =VectorHashKeyWrapperBatch.compileKeyWrapperBatch(keyExpressions);
@@ -298,4 +295,9 @@ public class VectorMapJoinOperator exten
batchIndex = -1;
keyValues = null;
}
+
+ @Override
+ public VectorizationContext getOuputVectorizationContext() {
+ return vOutContext;
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Wed Nov 27 20:11:44 2013
@@ -77,11 +77,8 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.UDFConv;
import org.apache.hadoop.hive.ql.udf.UDFHex;
-import org.apache.hadoop.hive.ql.udf.UDFOPNegative;
-import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
import org.apache.hadoop.hive.ql.udf.UDFToBoolean;
import org.apache.hadoop.hive.ql.udf.UDFToByte;
import org.apache.hadoop.hive.ql.udf.UDFToDouble;
@@ -355,22 +352,15 @@ public class VectorizationContext {
}
GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
- if (!(gudf instanceof GenericUDFBridge)) {
- return exprDesc;
- }
-
- Class<? extends UDF> cl = ((GenericUDFBridge) gudf).getUdfClass();
-
- if (cl.equals(UDFOPNegative.class) || cl.equals(UDFOPPositive.class)) {
+ if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive) {
ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
ObjectInspector output = evaluator.initialize(null);
-
Object constant = evaluator.evaluate(null);
Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
return new ExprNodeConstantDesc(java);
- } else {
- return exprDesc;
}
+
+ return exprDesc;
}
/* Fold simple unary expressions in all members of the input list and return new list
@@ -561,6 +551,8 @@ public class VectorizationContext {
return getBetweenFilterExpression(childExpr, mode);
} else if (udf instanceof GenericUDFIn) {
return getInFilterExpression(childExpr);
+ } else if (udf instanceof GenericUDFOPPositive) {
+ return getIdentityExpression(childExpr);
} else if (udf instanceof GenericUDFBridge) {
VectorExpression v = getGenericUDFBridgeVectorExpression((GenericUDFBridge) udf, childExpr, mode);
if (v != null) {
@@ -670,9 +662,7 @@ public class VectorizationContext {
private VectorExpression getGenericUDFBridgeVectorExpression(GenericUDFBridge udf,
List<ExprNodeDesc> childExpr, Mode mode) throws HiveException {
Class<? extends UDF> cl = udf.getUdfClass();
- if (cl.equals(UDFOPPositive.class)) {
- return getIdentityExpression(childExpr);
- } else if (isCastToIntFamily(cl)) {
+ if (isCastToIntFamily(cl)) {
return getCastToLongExpression(childExpr);
} else if (cl.equals(UDFToBoolean.class)) {
return getCastToBoolean(childExpr);
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java Wed Nov 27 20:11:44 2013
@@ -177,51 +177,6 @@ public class VectorizedRowBatch implemen
this.valueWriters = valueWriters;
}
- public static VectorizedRowBatch buildBatch(Map<Integer, String> typeMap,
- Map<String, Integer> columnMap) throws HiveException {
-
- Map<Integer, ColumnVector> mapVectorColumn = new HashMap<Integer, ColumnVector>(typeMap.size());
- int maxIndex = 0;
-
- Iterator<Entry<Integer, String>> typeMapIt = typeMap.entrySet().iterator();
- while(typeMapIt.hasNext()) {
- Entry<Integer, String> type = typeMapIt.next();
- ColumnVector cv = VectorizationContext.allocateColumnVector(type.getValue(),
- VectorizedRowBatch.DEFAULT_SIZE);
- mapVectorColumn.put(type.getKey(), cv);
- if (maxIndex < type.getKey()) {
- maxIndex = type.getKey();
- }
- }
-
- VectorizedRowBatch batch = new VectorizedRowBatch(maxIndex+1);
- for(int i=0; i <= maxIndex; ++i) {
- ColumnVector cv = mapVectorColumn.get(i);
- if (cv == null) {
- // allocate a default type for the unused column.
- // there are APIs that expect all cols[i] to be non NULL
- cv = VectorizationContext.allocateColumnVector("long",
- VectorizedRowBatch.DEFAULT_SIZE);
- }
- batch.cols[i] = cv;
- }
-
- // Validate that every column in the column map exists
- Iterator<Entry<String, Integer>> columnMapIt = columnMap.entrySet().iterator();
- while(columnMapIt.hasNext()) {
- Entry<String, Integer> cm = columnMapIt.next();
- if (batch.cols.length <= cm.getValue() || batch.cols[cm.getValue()] == null) {
- throw new HiveException(String.format(
- "Internal error: The type map has no entry for column %d %s",
- cm.getValue(), cm.getKey()));
- }
- }
-
- batch.reset();
-
- return batch;
- }
-
/**
* Resets the row batch to default state
* - sets selectedInUse to false
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java Wed Nov 27 20:11:44 2013
@@ -96,6 +96,25 @@ public class VectorizedRowBatchCtx {
public VectorizedRowBatchCtx() {
}
+
+ /**
+ * Initializes the VectorizedRowBatch context based on an arbitrary object inspector
+ * Used by non-tablescan operators when they change the vectorization context
+ * @param hiveConf
+ * @param fileKey
+ * The key on which to retrieve the extra column mapping from the map scratch
+ * @param rowOI
+ * Object inspector that shapes the column types
+ */
+ public void init(Configuration hiveConf, String fileKey,
+ StructObjectInspector rowOI) {
+ columnTypeMap = Utilities
+ .getMapRedWork(hiveConf).getMapWork().getScratchColumnVectorTypes()
+ .get(fileKey);
+ this.rowOI= rowOI;
+ this.rawRowOI = rowOI;
+ }
+
/**
* Initializes VectorizedRowBatch context based on the
@@ -251,6 +270,7 @@ public class VectorizedRowBatchCtx {
}
result.numCols = fieldRefs.size();
this.addScratchColumnsToBatch(result);
+ result.reset();
return result;
}
@@ -351,4 +371,5 @@ public class VectorizedRowBatchCtx {
return new LongColumnVector(defaultSize);
}
}
+
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Wed Nov 27 20:11:44 2013
@@ -302,8 +302,13 @@ public class HiveInputFormat<K extends W
// Make filter pushdown information available to getSplits.
if ((aliases != null) && (aliases.size() == 1)) {
Operator op = mrwork.getAliasToWork().get(aliases.get(0));
- if (op instanceof TableScanOperator) {
+ if ((op != null) && (op instanceof TableScanOperator)) {
tableScan = (TableScanOperator) op;
+ // push down projections.
+ ColumnProjectionUtils.appendReadColumns(
+ newjob, tableScan.getNeededColumnIDs(), tableScan.getNeededColumns());
+ // push down filters
+ pushFilters(newjob, tableScan);
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Wed Nov 27 20:11:44 2013
@@ -183,8 +183,7 @@ public class OrcInputFormat implements
public static SearchArgument createSarg(List<OrcProto.Type> types, Configuration conf) {
String serializedPushdown = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
if (serializedPushdown == null
- || (conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) == null
- && conf.get(serdeConstants.LIST_COLUMNS) == null)) {
+ || conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) == null) {
LOG.debug("No ORC pushdown predicate");
return null;
}
@@ -667,15 +666,21 @@ public class OrcInputFormat implements
int[] filterColumns = null;
if (sarg != null) {
List<PredicateLeaf> sargLeaves = null;
- String[] columnNames = conf.get(serdeConstants.LIST_COLUMNS).split(",");
- if (columnNames == null) {
- columnNames = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR).split(",");
- }
+ String[] allColumns = conf.get(serdeConstants.LIST_COLUMNS).split(",");
+ String[] neededColumns = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR).split(",");
sargLeaves = sarg.getLeaves();
filterColumns = new int[sargLeaves.size()];
for (int i = 0; i < filterColumns.length; ++i) {
String colName = sargLeaves.get(i).getColumnName();
- filterColumns[i] = RecordReaderImpl.findColumns(columnNames, colName);
+
+ // if needed columns does not contain the column specified in filter expression then
+ // it must be partition column. There will not be columns within ORC file for partitioned
+ // column, so we can ignore them
+ if (containsColumn(neededColumns, colName)) {
+ filterColumns[i] = RecordReaderImpl.findColumns(allColumns, colName);
+ } else {
+ filterColumns[i] = -1;
+ }
}
stripeStats = metadata.getStripeStatistics();
@@ -785,6 +790,15 @@ public class OrcInputFormat implements
}
}
+ private boolean containsColumn(String[] neededColumns, String colName) {
+ for (String col : neededColumns) {
+ if (colName.equalsIgnoreCase(col)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private boolean isStripeSatisfyPredicate(StripeStatistics stripeStatistics,
SearchArgument sarg, int[] filterColumns) {
if (sarg != null && filterColumns != null) {
@@ -799,6 +813,12 @@ public class OrcInputFormat implements
Object maxValue = getMax(stats);
truthValues[pred] = RecordReaderImpl.evaluatePredicateRange(predLeaves.get(pred),
minValue, maxValue);
+ } else {
+
+ // parition column case.
+ // partition filter will be evaluated by partition pruner so
+ // we will not evaluate partition filter here.
+ truthValues[pred] = TruthValue.YES_NO_NULL;
}
}
return sarg.evaluate(truthValues).isNeeded();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Wed Nov 27 20:11:44 2013
@@ -23,9 +23,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.Stack;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.mr
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
@@ -82,13 +85,11 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFAsin;
import org.apache.hadoop.hive.ql.udf.UDFAtan;
import org.apache.hadoop.hive.ql.udf.UDFBin;
-import org.apache.hadoop.hive.ql.udf.UDFCeil;
import org.apache.hadoop.hive.ql.udf.UDFConv;
import org.apache.hadoop.hive.ql.udf.UDFCos;
import org.apache.hadoop.hive.ql.udf.UDFDayOfMonth;
import org.apache.hadoop.hive.ql.udf.UDFDegrees;
import org.apache.hadoop.hive.ql.udf.UDFExp;
-import org.apache.hadoop.hive.ql.udf.UDFFloor;
import org.apache.hadoop.hive.ql.udf.UDFHex;
import org.apache.hadoop.hive.ql.udf.UDFHour;
import org.apache.hadoop.hive.ql.udf.UDFLTrim;
@@ -99,9 +100,6 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFLog10;
import org.apache.hadoop.hive.ql.udf.UDFLog2;
import org.apache.hadoop.hive.ql.udf.UDFMinute;
-import org.apache.hadoop.hive.ql.udf.UDFOPNegative;
-import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
-import org.apache.hadoop.hive.ql.udf.UDFPower;
import org.apache.hadoop.hive.ql.udf.UDFRTrim;
import org.apache.hadoop.hive.ql.udf.UDFRadians;
import org.apache.hadoop.hive.ql.udf.UDFRand;
@@ -127,7 +125,9 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCeil;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFConcat;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFloor;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLower;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
@@ -140,11 +140,14 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMinus;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMod;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMultiply;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNegative;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPositive;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFPower;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFRound;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFPosMod;
@@ -180,13 +183,13 @@ public class Vectorizer implements Physi
supportedDataTypes.add("float");
supportedDataTypes.add("double");
- supportedGenericUDFs.add(UDFOPNegative.class);
- supportedGenericUDFs.add(UDFOPPositive.class);
supportedGenericUDFs.add(GenericUDFOPPlus.class);
supportedGenericUDFs.add(GenericUDFOPMinus.class);
supportedGenericUDFs.add(GenericUDFOPMultiply.class);
supportedGenericUDFs.add(GenericUDFOPDivide.class);
supportedGenericUDFs.add(GenericUDFOPMod.class);
+ supportedGenericUDFs.add(GenericUDFOPNegative.class);
+ supportedGenericUDFs.add(GenericUDFOPPositive.class);
supportedGenericUDFs.add(GenericUDFOPEqualOrLessThan.class);
supportedGenericUDFs.add(GenericUDFOPEqualOrGreaterThan.class);
@@ -224,14 +227,14 @@ public class Vectorizer implements Physi
supportedGenericUDFs.add(UDFAtan.class);
supportedGenericUDFs.add(UDFDegrees.class);
supportedGenericUDFs.add(UDFRadians.class);
- supportedGenericUDFs.add(UDFFloor.class);
- supportedGenericUDFs.add(UDFCeil.class);
+ supportedGenericUDFs.add(GenericUDFFloor.class);
+ supportedGenericUDFs.add(GenericUDFCeil.class);
supportedGenericUDFs.add(UDFExp.class);
supportedGenericUDFs.add(UDFLn.class);
supportedGenericUDFs.add(UDFLog2.class);
supportedGenericUDFs.add(UDFLog10.class);
supportedGenericUDFs.add(UDFLog.class);
- supportedGenericUDFs.add(UDFPower.class);
+ supportedGenericUDFs.add(GenericUDFPower.class);
supportedGenericUDFs.add(GenericUDFRound.class);
supportedGenericUDFs.add(GenericUDFPosMod.class);
supportedGenericUDFs.add(UDFSqrt.class);
@@ -354,8 +357,17 @@ public class Vectorizer implements Physi
topNodes.addAll(mapWork.getAliasToWork().values());
HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutput);
- mapWork.setScratchColumnVectorTypes(vnp.getScratchColumnVectorTypes());
- mapWork.setScratchColumnMap(vnp.getScratchColumnMap());
+
+ Map<String, Map<Integer, String>> columnVectorTypes = vnp.getScratchColumnVectorTypes();
+ mapWork.setScratchColumnVectorTypes(columnVectorTypes);
+ Map<String, Map<String, Integer>> columnMap = vnp.getScratchColumnMap();
+ mapWork.setScratchColumnMap(columnMap);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("vectorTypes: %s", columnVectorTypes.toString()));
+ LOG.debug(String.format("columnMap: %s", columnMap.toString()));
+ }
+
return;
}
}
@@ -423,33 +435,42 @@ public class Vectorizer implements Physi
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- Node firstOp = stack.firstElement();
- TableScanOperator tsOp = null;
-
- tsOp = (TableScanOperator) firstOp;
-
- VectorizationContext vContext = vContextsByTSOp.get(tsOp);
- if (vContext == null) {
- String fileKey = "";
+ Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
+
+ VectorizationContext vContext = null;
+
+ if (op instanceof TableScanOperator) {
+ vContext = getVectorizationContext(op, physicalContext);
for (String onefile : mWork.getPathToAliases().keySet()) {
List<String> aliases = mWork.getPathToAliases().get(onefile);
for (String alias : aliases) {
- Operator<? extends OperatorDesc> op = mWork.getAliasToWork().get(alias);
- if (op == tsOp) {
- fileKey = onefile;
- if (vContext == null) {
- vContext = getVectorizationContext(tsOp, physicalContext);
- }
- vContext.setFileKey(fileKey);
- vectorizationContexts.put(fileKey, vContext);
+ Operator<? extends OperatorDesc> opRoot = mWork.getAliasToWork().get(alias);
+ if (op == opRoot) {
+ // The same vectorization context is copied multiple times into
+ // the MapWork scratch columnMap
+ // Each partition gets a copy
+ //
+ vContext.setFileKey(onefile);
+ vectorizationContexts.put(onefile, vContext);
break;
}
}
}
- vContextsByTSOp.put(tsOp, vContext);
+ vContextsByTSOp.put(op, vContext);
+ } else {
+ assert stack.size() > 1;
+ // Walk down the stack of operators until we found one willing to give us a context.
+ // At the bottom will be the TS operator, guaranteed to have a context
+ int i= stack.size()-2;
+ while (vContext == null) {
+ Operator<? extends OperatorDesc> opParent = (Operator<? extends OperatorDesc>) stack.get(i);
+ vContext = vContextsByTSOp.get(opParent);
+ --i;
+ }
}
-
- Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
+
+ assert vContext != null;
+
if (op.getType().equals(OperatorType.REDUCESINK) &&
op.getParentOperators().get(0).getType().equals(OperatorType.GROUPBY)) {
// No need to vectorize
@@ -465,6 +486,12 @@ public class Vectorizer implements Physi
if (vectorOp != op) {
opsDone.add(vectorOp);
}
+ if (vectorOp instanceof VectorizationContextRegion) {
+ VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp;
+ VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext();
+ vContextsByTSOp.put(op, vOutContext);
+ vectorizationContexts.put(vOutContext.getFileKey(), vOutContext);
+ }
}
} catch (HiveException e) {
throw new SemanticException(e);
@@ -690,7 +717,7 @@ public class Vectorizer implements Physi
return supportedDataTypes.contains(type.toLowerCase());
}
- private VectorizationContext getVectorizationContext(TableScanOperator op,
+ private VectorizationContext getVectorizationContext(Operator<? extends OperatorDesc> op,
PhysicalContext pctx) {
RowResolver rr = pctx.getParseContext().getOpParseCtx().get(op).getRowResolver();
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java Wed Nov 27 20:11:44 2013
@@ -85,8 +85,6 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.UDFLTrim;
import org.apache.hadoop.hive.ql.udf.UDFLog;
-import org.apache.hadoop.hive.ql.udf.UDFOPNegative;
-import org.apache.hadoop.hive.ql.udf.UDFPower;
import org.apache.hadoop.hive.ql.udf.UDFSin;
import org.apache.hadoop.hive.ql.udf.UDFYear;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -101,10 +99,12 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMinus;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMod;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMultiply;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNegative;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFPower;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFRound;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp;
@@ -619,7 +619,7 @@ public class TestVectorizationContext {
public void testUnaryMinusColumnLong() throws HiveException {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false);
ExprNodeGenericFuncDesc negExprDesc = new ExprNodeGenericFuncDesc();
- GenericUDF gudf = new GenericUDFBridge("-", true, UDFOPNegative.class.getCanonicalName());
+ GenericUDF gudf = new GenericUDFOPNegative();
negExprDesc.setGenericUDF(gudf);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
children.add(col1Expr);
@@ -637,7 +637,7 @@ public class TestVectorizationContext {
public void testUnaryMinusColumnDouble() throws HiveException {
ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Float.class, "col1", "table", false);
ExprNodeGenericFuncDesc negExprDesc = new ExprNodeGenericFuncDesc();
- GenericUDF gudf = new GenericUDFBridge("-", true, UDFOPNegative.class.getCanonicalName());
+ GenericUDF gudf = new GenericUDFOPNegative();
negExprDesc.setGenericUDF(gudf);
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(1);
children.add(col1Expr);
@@ -841,11 +841,10 @@ public class TestVectorizationContext {
Assert.assertTrue(4.5 == ((FuncLogWithBaseLongToDouble) ve).getBase());
//Power with double power
- gudfBridge = new GenericUDFBridge("power", false, UDFPower.class.getName());
children2.clear();
children2.add(colDesc2);
children2.add(new ExprNodeConstantDesc(4.5));
- mathFuncExpr.setGenericUDF(gudfBridge);
+ mathFuncExpr.setGenericUDF(new GenericUDFPower());
mathFuncExpr.setChildren(children2);
ve = vc.getVectorExpression(mathFuncExpr);
Assert.assertEquals(FuncPowerDoubleToDouble.class, ve.getClass());
Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java Wed Nov 27 20:11:44 2013
@@ -19,6 +19,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.io.Text;
@@ -64,8 +65,11 @@ public class TestOrcSplitElimination {
@Before
public void openFileSystem() throws Exception {
conf = new JobConf();
+ // all columns
conf.set("columns", "userid,string1,subtype,decimal1,ts");
conf.set("columns.types", "bigint,string,double,decimal,timestamp");
+ // needed columns
+ conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, "userid,subtype");
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir, "TestOrcFile." +
testCaseName.getMethodName() + ".orc");
Modified: hive/branches/tez/ql/src/test/queries/clientpositive/orc_split_elimination.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/orc_split_elimination.q?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/orc_split_elimination.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/orc_split_elimination.q Wed Nov 27 20:11:44 2013
@@ -99,3 +99,70 @@ SET hive.optimize.index.filter=true;
-- 5 mappers
select userid,string1,subtype,decimal1,ts from orc_split_elim where userid<=70 order by userid;
SET hive.optimize.index.filter=false;
+
+-- partitioned table
+create table orc_split_elim_part (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (country string, year int) stored as orc;
+
+alter table orc_split_elim_part add partition(country='us', year=2000);
+alter table orc_split_elim_part add partition(country='us', year=2001);
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_part partition(country='us', year=2000);
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_part partition(country='us', year=2001);
+
+-- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us'order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' order by userid;
+SET hive.optimize.index.filter=false;
+
+-- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001) order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001) order by userid;
+SET hive.optimize.index.filter=false;
+
+-- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000 order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 1 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000 order by userid;
+SET hive.optimize.index.filter=false;
+
+-- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 4 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' order by userid;
+SET hive.optimize.index.filter=false;
+
+-- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001) order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 4 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001) order by userid;
+SET hive.optimize.index.filter=false;
+
+-- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000 order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000 order by userid;
+SET hive.optimize.index.filter=false;
+
+-- 0 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in' order by userid;
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002 order by userid;
+
+SET hive.optimize.index.filter=true;
+-- 0 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in' order by userid;
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002 order by userid;
+SET hive.optimize.index.filter=false;
Modified: hive/branches/tez/ql/src/test/results/clientpositive/decimal_udf.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/decimal_udf.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/decimal_udf.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/decimal_udf.q.out Wed Nov 27 20:11:44 2013
@@ -1596,7 +1596,7 @@ STAGE PLANS:
Select Operator
expressions:
expr: ceil(key)
- type: decimal(38,18)
+ type: decimal(21,0)
outputColumnNames: _col0
ListSink
@@ -1669,7 +1669,7 @@ STAGE PLANS:
Select Operator
expressions:
expr: floor(key)
- type: decimal(38,18)
+ type: decimal(21,0)
outputColumnNames: _col0
ListSink
@@ -1815,7 +1815,7 @@ STAGE PLANS:
Select Operator
expressions:
expr: power(key, 2)
- type: decimal(38,18)
+ type: double
outputColumnNames: _col0
ListSink
@@ -1828,44 +1828,44 @@ POSTHOOK: query: SELECT POWER(key, 2) FR
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_udf
#### A masked pattern was here ####
-19360000
+1.936E7
NULL
-0
-0
-10000
-100
-1
-0.01
-0.0001
-40000
-400
-4
-0
-0.04
-0.0004
+0.0
+0.0
+10000.0
+100.0
+1.0
+0.010000000000000002
+1.0E-4
+40000.0
+400.0
+4.0
+0.0
+0.04000000000000001
+4.0E-4
0.09
-0.1089
-0.110889
+0.10890000000000001
+0.11088900000000002
0.09
-0.1089
-0.110889
-1
-4
+0.10890000000000001
+0.11088900000000002
+1.0
+4.0
9.8596
-1.2544
-1.2544
-1.258884
-1.2544
-1.258884
-15376
+1.2544000000000002
+1.2544000000000002
+1.2588840000000003
+1.2544000000000002
+1.2588840000000003
+15376.0
15675.04
1576255.1401
9.8596
9.8596
9.8596
-1
-1524157875323883675.019051998750190521
-1524157875323883652.7968299765279684
+1.0
+1.52415787532388352E18
+1.52415787532388352E18
PREHOOK: query: -- modulo
EXPLAIN SELECT (key + 1) % (key / 2) FROM DECIMAL_UDF
PREHOOK: type: QUERY
Modified: hive/branches/tez/ql/src/test/results/clientpositive/literal_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/literal_decimal.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/literal_decimal.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/literal_decimal.q.out Wed Nov 27 20:11:44 2013
@@ -18,7 +18,7 @@ STAGE PLANS:
Select Operator
expressions:
expr: (- 1)
- type: decimal(38,18)
+ type: decimal(1,0)
expr: 0
type: decimal(1,0)
expr: 1
@@ -26,7 +26,7 @@ STAGE PLANS:
expr: 3.14
type: decimal(3,2)
expr: (- 3.14)
- type: decimal(38,18)
+ type: decimal(3,2)
expr: 99999999999999999
type: decimal(17,0)
expr: 99999999999999999.9999999999999
Modified: hive/branches/tez/ql/src/test/results/clientpositive/orc_split_elimination.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/orc_split_elimination.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/orc_split_elimination.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/orc_split_elimination.q.out Wed Nov 27 20:11:44 2013
@@ -255,3 +255,262 @@ POSTHOOK: Input: default@orc_split_elim
13 bar 80.0 2 1969-12-31 16:00:05
29 cat 8.0 3 1969-12-31 16:00:10
70 dog 1.8 4 1969-12-31 16:00:15
+PREHOOK: query: -- partitioned table
+create table orc_split_elim_part (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (country string, year int) stored as orc
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- partitioned table
+create table orc_split_elim_part (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (country string, year int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_split_elim_part
+PREHOOK: query: alter table orc_split_elim_part add partition(country='us', year=2000)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@orc_split_elim_part
+POSTHOOK: query: alter table orc_split_elim_part add partition(country='us', year=2000)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Output: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: query: alter table orc_split_elim_part add partition(country='us', year=2001)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@orc_split_elim_part
+POSTHOOK: query: alter table orc_split_elim_part add partition(country='us', year=2001)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Output: default@orc_split_elim_part@country=us/year=2001
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_part partition(country='us', year=2000)
+PREHOOK: type: LOAD
+PREHOOK: Output: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_part partition(country='us', year=2000)
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_part partition(country='us', year=2001)
+PREHOOK: type: LOAD
+PREHOOK: Output: default@orc_split_elim_part@country=us/year=2001
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_split_elim_part partition(country='us', year=2001)
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@orc_split_elim_part@country=us/year=2001
+PREHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us'order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us'order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001) order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001) order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001) order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and (year=2000 or year=2001) order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000 order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+POSTHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000 order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- 1 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000 order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=2 and country='us' and year=2000 order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+PREHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+5 eat 0.8 6 1969-12-31 16:00:20
+5 eat 0.8 6 1969-12-31 16:00:20
+PREHOOK: query: -- 4 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 4 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+5 eat 0.8 6 1969-12-31 16:00:20
+5 eat 0.8 6 1969-12-31 16:00:20
+PREHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001) order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001) order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+5 eat 0.8 6 1969-12-31 16:00:20
+5 eat 0.8 6 1969-12-31 16:00:20
+PREHOOK: query: -- 4 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001) order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+POSTHOOK: query: -- 4 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and (year=2000 or year=2001) order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2001
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+2 foo 0.8 1 1969-12-31 16:00:00
+5 eat 0.8 6 1969-12-31 16:00:20
+5 eat 0.8 6 1969-12-31 16:00:20
+PREHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000 order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+POSTHOOK: query: -- 10 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000 order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+5 eat 0.8 6 1969-12-31 16:00:20
+PREHOOK: query: -- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000 order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+PREHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=5 and country='us' and year=2000 order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+POSTHOOK: Input: default@orc_split_elim_part@country=us/year=2000
+#### A masked pattern was here ####
+2 foo 0.8 1 1969-12-31 16:00:00
+5 eat 0.8 6 1969-12-31 16:00:20
+PREHOOK: query: -- 0 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in' order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 0 mapper - no split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in' order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+PREHOOK: query: select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002 order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+POSTHOOK: query: select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002 order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+PREHOOK: query: -- 0 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in' order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+POSTHOOK: query: -- 0 mapper - split elimination
+select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='in' order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+PREHOOK: query: select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002 order by userid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
+POSTHOOK: query: select userid,string1,subtype,decimal1,ts from orc_split_elim_part where userid<=70 and country='us' and year=2002 order by userid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_split_elim_part
+#### A masked pattern was here ####
Modified: hive/branches/tez/ql/src/test/results/clientpositive/udf4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/udf4.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/udf4.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/udf4.q.out Wed Nov 27 20:11:44 2013
@@ -103,7 +103,7 @@ STAGE PLANS:
type: bigint
expr: ceil((- 1.5))
type: bigint
- expr: ceiling(1.0)
+ expr: ceil(1.0)
type: bigint
expr: rand(3)
type: double
Modified: hive/branches/tez/ql/src/test/results/clientpositive/udf7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/udf7.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/udf7.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/udf7.q.out Wed Nov 27 20:11:44 2013
@@ -85,7 +85,7 @@ STAGE PLANS:
type: double
expr: round(exp(2.0), 12)
type: double
- expr: pow(2, 3)
+ expr: power(2, 3)
type: double
expr: power(2, 3)
type: double
@@ -100,11 +100,11 @@ STAGE PLANS:
expr: power((- 1), 2)
type: double
expr: power(CAST( 1 AS decimal(10,0)), 0)
- type: decimal(38,18)
+ type: double
expr: power(CAST( 2 AS decimal(10,0)), 3)
- type: decimal(38,18)
- expr: pow(CAST( 2 AS decimal(10,0)), 3)
- type: decimal(38,18)
+ type: double
+ expr: power(CAST( 2 AS decimal(10,0)), 3)
+ type: double
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27
File Output Operator
compressed: false
@@ -142,4 +142,4 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
POSTHOOK: Lineage: dest1.c1 SIMPLE []
-1.098612288668 NULL NULL 1.098612288668 NULL NULL 1.584962500721 NULL NULL 0.47712125472 NULL NULL 1.584962500721 NULL NULL NULL -1.0 7.389056098931 8.0 8.0 0.125 8.0 2.0 NaN 1.0 1 8 8
+1.098612288668 NULL NULL 1.098612288668 NULL NULL 1.584962500721 NULL NULL 0.47712125472 NULL NULL 1.584962500721 NULL NULL NULL -1.0 7.389056098931 8.0 8.0 0.125 8.0 2.0 NaN 1.0 1.0 8.0 8.0
Modified: hive/branches/tez/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/vectorization_short_regress.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/vectorization_short_regress.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/vectorization_short_regress.q.out Wed Nov 27 20:11:44 2013
@@ -560,7 +560,7 @@ STAGE PLANS:
expr: _col7
type: double
expr: (- (_col0 / (- 3728)))
- type: decimal(38,18)
+ type: decimal(16,6)
expr: ((- (_col0 * (- 3728))) % ((- 563) % (_col0 * (- 3728))))
type: int
expr: ((_col0 / (- 3728)) - _col4)
@@ -2740,7 +2740,7 @@ STAGE PLANS:
expr: (cbigint % 79.553)
type: double
expr: (- (cint / cbigint))
- type: decimal(38,18)
+ type: decimal(30,20)
expr: (10.175 % cfloat)
type: double
expr: (- cfloat)
@@ -2855,8 +2855,8 @@ WHERE (((197 > ctinyint)
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
--462839731 988888 ss false -51.0 NULL NULL NULL Lml5J2QBU77 false -468.04059812638033831941 44.210000000033915 468.040598126380338319 10.175 51.0 -102.0 -102.0 NULL NULL -988888 417.04059812638036 NULL 3569 NULL NULL
--635141101 -89010 ss false -51.0 NULL NULL NULL rVWAj4N1MCg8Scyp7wj2C true 7135.61511066172340186496 -69.74600000000305 -7135.615110661723401865 10.175 51.0 -102.0 -102.0 NULL NULL 89010 -7186.6151106617235 NULL 3569 NULL NULL
+-462839731 988888 ss false -51.0 NULL NULL NULL Lml5J2QBU77 false -468.04059812638033831941 44.210000000033915 468.04059812638033831941 10.175 51.0 -102.0 -102.0 NULL NULL -988888 417.04059812638036 NULL 3569 NULL NULL
+-635141101 -89010 ss false -51.0 NULL NULL NULL rVWAj4N1MCg8Scyp7wj2C true 7135.61511066172340186496 -69.74600000000305 -7135.61511066172340186496 10.175 51.0 -102.0 -102.0 NULL NULL 89010 -7186.6151106617235 NULL 3569 NULL NULL
WARNING: Comparing a bigint and a double may result in a loss of precision.
PREHOOK: query: -- TargetTypeClasses: String, Bool, Double, Long, Timestamp
-- Functions: Sum, Max, Avg, Var, StDevP, VarP
Modified: hive/branches/tez/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out?rev=1546180&r1=1546179&r2=1546180&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out Wed Nov 27 20:11:44 2013
@@ -157,7 +157,7 @@ STAGE PLANS:
type: double
expr: log(2.0, cdouble)
type: double
- expr: pow(log2(cdouble), 2.0)
+ expr: power(log2(cdouble), 2.0)
type: double
expr: power(log2(cdouble), 2.0)
type: double