You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/12/02 18:06:50 UTC
svn commit: r1642955 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
lib/ optimizer/optiq/translator/ parse/
Author: hashutosh
Date: Tue Dec 2 17:06:50 2014
New Revision: 1642955
URL: http://svn.apache.org/r1642955
Log:
HIVE-8922 : CBO: assorted date and timestamp issues (Sergey Shelukhin via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/TaskGraphWalker.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/TaskGraphWalker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/TaskGraphWalker.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/TaskGraphWalker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/TaskGraphWalker.java Tue Dec 2 17:06:50 2014
@@ -112,20 +112,6 @@ public class TaskGraphWalker implements
retMap.put(nd, retVal);
}
- public void dispatch(Node nd, Stack<Node> ndStack) throws SemanticException {
- Object[] nodeOutputs = null;
- if (nd.getChildren() != null) {
- nodeOutputs = new Object[nd.getChildren().size()];
- int i = 1;
- for (Node child : nd.getChildren()) {
- nodeOutputs[i++] = retMap.get(child);
- }
- }
-
- Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs);
- retMap.put(nd, retVal);
- }
-
/**
* starting point for walking.
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java Tue Dec 2 17:06:50 2014
@@ -213,7 +213,7 @@ class ASTBuilder {
case TIME:
case TIMESTAMP: {
val = literal.getValue();
- type = HiveParser.TOK_TIMESTAMP;
+ type = HiveParser.TOK_TIMESTAMPLITERAL;
DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
val = df.format(((Calendar) val).getTime());
val = "'" + val + "'";
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ExprNodeConverter.java Tue Dec 2 17:06:50 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer.optiq.translator;
import java.sql.Date;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.LinkedList;
@@ -29,6 +30,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.eigenbase.reltype.RelDataType;
import org.eigenbase.reltype.RelDataTypeField;
@@ -98,8 +100,13 @@ public class ExprNodeConverter extends R
SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(), 2), tmpExprArgs);
}
} else {
- gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()),
- SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(), args.size()), args);
+ GenericUDF hiveUdf = SqlFunctionConverter.getHiveUDF(
+ call.getOperator(), call.getType(), args.size());
+ if (hiveUdf == null) {
+ throw new RuntimeException("Cannot find UDF for " + call.getType() + " " + call.getOperator()
+ + "[" + call.getOperator().getKind() + "]/" + args.size());
+ }
+ gfDesc = new ExprNodeGenericFuncDesc(TypeConverter.convert(call.getType()), hiveUdf, args);
}
return gfDesc;
@@ -134,8 +141,13 @@ public class ExprNodeConverter extends R
case DATE:
return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo,
new Date(((Calendar)literal.getValue()).getTimeInMillis()));
- case TIMESTAMP:
- return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, literal.getValue3());
+ case TIMESTAMP: {
+ Object value = literal.getValue3();
+ if (value instanceof Long) {
+ value = new Timestamp((Long)value);
+ }
+ return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, value);
+ }
case BINARY:
return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, literal.getValue3());
case DECIMAL:
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java Tue Dec 2 17:06:50 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimi
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@@ -383,8 +384,14 @@ public class RexNodeConverter {
optiqLiteral = rexBuilder.makeDateLiteral(cal);
break;
case TIMESTAMP:
- optiqLiteral = rexBuilder.makeTimestampLiteral((Calendar) value,
- RelDataType.PRECISION_NOT_SPECIFIED);
+ Calendar c = null;
+ if (value instanceof Calendar) {
+ c = (Calendar)value;
+ } else {
+ c = Calendar.getInstance();
+ c.setTimeInMillis(((Timestamp)value).getTime());
+ }
+ optiqLiteral = rexBuilder.makeTimestampLiteral(c, RelDataType.PRECISION_NOT_SPECIFIED);
break;
case BINARY:
case VOID:
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java Tue Dec 2 17:06:50 2014
@@ -119,13 +119,14 @@ public class SqlFunctionConverter {
LOG.warn("Failed to load udf " + name, e);
hFn = null;
}
- if (hFn == null)
+ if (hFn == null) {
try {
hFn = handleExplicitCast(op, dt);
} catch (SemanticException e) {
LOG.warn("Failed to load udf " + name, e);
hFn = null;
}
+ }
return hFn == null ? null : hFn.getGenericUDF();
}
@@ -158,8 +159,8 @@ public class SqlFunctionConverter {
castUDF = FunctionRegistry.getFunctionInfo("double");
} else if (castType.equals(TypeInfoFactory.timestampTypeInfo)) {
castUDF = FunctionRegistry.getFunctionInfo("timestamp");
- } else if (castType.equals(TypeInfoFactory.dateTypeInfo)) {
- castUDF = FunctionRegistry.getFunctionInfo("datetime");
+ } else if (castType.equals(TypeInfoFactory.dateTypeInfo)) {
+ castUDF = FunctionRegistry.getFunctionInfo("date");
} else if (castType instanceof DecimalTypeInfo) {
castUDF = handleCastForParameterizedType(castType,
FunctionRegistry.getFunctionInfo("decimal"));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java Tue Dec 2 17:06:50 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.parse;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
/**
@@ -25,6 +27,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
* phase in query compilation.
*/
public class TypeCheckCtx implements NodeProcessorCtx {
+ protected static final Log LOG = LogFactory.getLog(TypeCheckCtx.class);
/**
* The row resolver of the previous operator. This field is used to generate
@@ -144,6 +147,11 @@ public class TypeCheckCtx implements Nod
*
*/
public void setError(String error, ASTNode errorSrcNode) {
+ if (LOG.isDebugEnabled()) {
+ // Log the callstack from which the error has been set.
+ LOG.debug("Setting error: [" + error + "] from "
+ + ((errorSrcNode == null) ? "null" : errorSrcNode.toStringTree()), new Exception());
+ }
this.error = error;
this.errorSrcNode = errorSrcNode;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1642955&r1=1642954&r2=1642955&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Dec 2 17:06:50 2014
@@ -1123,9 +1123,8 @@ public class TypeCheckProcFactory {
// Create all children
int childrenBegin = (isFunction ? 1 : 0);
- ArrayList<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(expr
- .getChildCount()
- - childrenBegin);
+ ArrayList<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(
+ expr.getChildCount() - childrenBegin);
for (int ci = childrenBegin; ci < expr.getChildCount(); ci++) {
if (nodeOutputs[ci] instanceof ExprNodeColumnListDesc) {
children.addAll(((ExprNodeColumnListDesc)nodeOutputs[ci]).getChildren());