You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/08/21 22:26:45 UTC
svn commit: r1619565 - in /hive/branches/cbo/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/
java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/
test/results/clientpositive/
Author: hashutosh
Date: Thu Aug 21 20:26:45 2014
New Revision: 1619565
URL: http://svn.apache.org/r1619565
Log:
HIVE-7827 : [CBO] null expr in select list is not handled correctly (Ashutosh Chauhan)
Modified:
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java Thu Aug 21 20:26:45 2014
@@ -108,7 +108,7 @@ class ASTBuilder {
static ASTNode selectExpr(ASTNode expr, String alias) {
return ASTBuilder.construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr)
- .add(HiveParser.Identifier, alias).node();
+ .add(HiveParser.Identifier, alias).node();
}
static ASTNode literal(RexLiteral literal) {
@@ -142,6 +142,9 @@ class ASTBuilder {
type = ((Boolean) val).booleanValue() ? HiveParser.KW_TRUE
: HiveParser.KW_FALSE;
break;
+ case NULL:
+ type = HiveParser.TOK_NULL;
+ break;
default:
throw new RuntimeException("Unsupported Type: " + sqlType);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java Thu Aug 21 20:26:45 2014
@@ -123,15 +123,23 @@ public class ASTConverter {
/*
* 6. Project
*/
- int i = 0;
- ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT");
+ if (!select.getChildExps().isEmpty()) {
- for (RexNode r : select.getChildExps()) {
- ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(new RexVisitor(schema)), select
- .getRowType().getFieldNames().get(i++));
- b.add(selectExpr);
+ ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT");
+ int i = 0;
+
+ for (RexNode r : select.getChildExps()) {
+ ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(new RexVisitor(schema)), select
+ .getRowType().getFieldNames().get(i++));
+ b.add(selectExpr);
+ }
+ hiveAST.select = b.node();
+ } else {
+ //TODO: We should never be here. But we will be for select null from t1.
+ // Once you figure out why, uncomment following line:
+ // throw new IllegalStateException("why am I here?");
}
- hiveAST.select = b.node();
+
/*
* 7. Order Use in Order By from the block above. RelNode has no pointer to
@@ -347,13 +355,13 @@ public class ASTConverter {
ASTNode wRangeAst = null;
ASTNode startAST = null;
- RexWindowBound ub = (RexWindowBound) window.getUpperBound();
+ RexWindowBound ub = window.getUpperBound();
if (ub != null) {
startAST = getWindowBound(ub);
}
ASTNode endAST = null;
- RexWindowBound lb = (RexWindowBound) window.getLowerBound();
+ RexWindowBound lb = window.getLowerBound();
if (lb != null) {
endAST = getWindowBound(lb);
}
@@ -496,7 +504,7 @@ public class ASTConverter {
* 1. ProjectRel will always be child of SortRel.<br>
* 2. In Optiq every projection in ProjectRelBase is uniquely named
* (unambigous) without using table qualifier (table name).<br>
- *
+ *
* @param order
* Hive Sort Rel Node
* @return Schema
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java Thu Aug 21 20:26:45 2014
@@ -15,11 +15,11 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate;
@@ -40,7 +40,6 @@ import org.eigenbase.rex.RexNode;
import org.eigenbase.rex.RexUtil;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.sql.fun.SqlCastFunction;
-import org.eigenbase.sql.type.SqlTypeName;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
@@ -81,6 +80,10 @@ public class RexNodeConverter {
}
public RexNode convert(ExprNodeDesc expr) throws SemanticException {
+ if (expr instanceof ExprNodeNullDesc) {
+ return m_cluster.getRexBuilder().makeNullLiteral(TypeConverter.convert(
+ expr.getTypeInfo(), m_cluster.getRexBuilder().getTypeFactory()).getSqlTypeName());
+ }
if (expr instanceof ExprNodeGenericFuncDesc) {
return convert((ExprNodeGenericFuncDesc) expr);
} else if (expr instanceof ExprNodeConstantDesc) {
@@ -90,8 +93,7 @@ public class RexNodeConverter {
} else {
throw new RuntimeException("Unsupported Expression");
}
- // TODO: handle a) ExprNodeNullDesc b) ExprNodeFieldDesc c)
- // ExprNodeColumnListDesc
+ // TODO: handle a) ExprNodeFieldDesc b) ExprNodeColumnListDesc
}
private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticException {
@@ -230,9 +232,10 @@ public class RexNodeConverter {
RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory);
PrimitiveCategory hiveTypeCategory = hiveType.getPrimitiveCategory();
- RexNode optiqLiteral = null;
+
Object value = literal.getValue();
+ RexNode optiqLiteral = null;
// TODO: Verify if we need to use ConstantObjectInspector to unwrap data
switch (hiveTypeCategory) {
case BOOLEAN:
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java Thu Aug 21 20:26:45 2014
@@ -110,8 +110,7 @@ public class TypeConverter {
switch (type.getPrimitiveCategory()) {
case VOID:
- // @todo: followup on VOID type in hive
- convertedType = dtFactory.createSqlType(SqlTypeName.OTHER);
+ convertedType = dtFactory.createSqlType(SqlTypeName.NULL);
break;
case BOOLEAN:
convertedType = dtFactory.createSqlType(SqlTypeName.BOOLEAN);
@@ -135,6 +134,7 @@ public class TypeConverter {
convertedType = dtFactory.createSqlType(SqlTypeName.DOUBLE);
break;
case STRING:
+ //TODO: shall we pass -1 for len to distinguish between STRING & VARCHAR on way out
convertedType = dtFactory.createSqlType(SqlTypeName.VARCHAR, 1);
break;
case DATE:
@@ -163,6 +163,10 @@ public class TypeConverter {
break;
}
+ if (null == convertedType) {
+ throw new RuntimeException("Unsupported Type : " + type.getTypeName());
+ }
+
return convertedType;
}
@@ -184,6 +188,7 @@ public class TypeConverter {
List<RelDataType> fTypes = Lists.transform(
structType.getAllStructFieldTypeInfos(),
new Function<TypeInfo, RelDataType>() {
+ @Override
public RelDataType apply(TypeInfo tI) {
return convert(tI, dtFactory);
}
@@ -197,7 +202,7 @@ public class TypeConverter {
// @todo what do we about unions?
throw new UnsupportedOperationException();
}
-
+
public static TypeInfo convert(RelDataType rType) {
if ( rType.isStruct() ) {
return convertStructType(rType);
@@ -209,11 +214,12 @@ public class TypeConverter {
return convertPrimtiveType(rType);
}
}
-
+
public static TypeInfo convertStructType(RelDataType rType) {
List<TypeInfo> fTypes = Lists.transform(
rType.getFieldList(),
new Function<RelDataTypeField, TypeInfo>() {
+ @Override
public TypeInfo apply(RelDataTypeField f) {
return convert(f.getType());
}
@@ -221,22 +227,23 @@ public class TypeConverter {
List<String> fNames = Lists.transform(
rType.getFieldList(),
new Function<RelDataTypeField, String>() {
+ @Override
public String apply(RelDataTypeField f) {
return f.getName();
}
});
return TypeInfoFactory.getStructTypeInfo(fNames, fTypes);
}
-
+
public static TypeInfo convertMapType(RelDataType rType) {
- return TypeInfoFactory.getMapTypeInfo(convert(rType.getKeyType()),
+ return TypeInfoFactory.getMapTypeInfo(convert(rType.getKeyType()),
convert(rType.getValueType()));
}
-
+
public static TypeInfo convertListType(RelDataType rType) {
return TypeInfoFactory.getListTypeInfo(convert(rType.getComponentType()));
}
-
+
public static TypeInfo convertPrimtiveType(RelDataType rType) {
switch(rType.getSqlTypeName()) {
case BOOLEAN:
@@ -269,7 +276,7 @@ public class TypeConverter {
default:
return TypeInfoFactory.voidTypeInfo;
}
-
+
}
/*********************** Convert Optiq Types To Hive Types ***********************/
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Aug 21 20:26:45 2014
@@ -13147,14 +13147,8 @@ public class SemanticAnalyzer extends Ba
}
selectStar = selectStar && exprList.getChildCount() == posn + 1;
- // 7. Replace NULL with CAST(NULL AS STRING)
ArrayList<String> columnNames = new ArrayList<String>();
for (int i = 0; i < col_list.size(); i++) {
- // Replace NULL with CAST(NULL AS STRING)
- if (col_list.get(i) instanceof ExprNodeNullDesc) {
- col_list.set(i, new ExprNodeConstantDesc(
- TypeInfoFactory.stringTypeInfo, null));
- }
columnNames.add(getColumnInternalName(i));
}
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q Thu Aug 21 20:26:45 2014
@@ -215,4 +215,5 @@ select * from (select c_int, b, t1.c fro
select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0) group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0) group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0) group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0) group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 >= 0) and (b > 0 or a >= 0) group by a, c having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
+-- 13. null expr in select list
+select null from t3;
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out Thu Aug 21 20:26:45 2014
@@ -17849,3 +17849,33 @@ POSTHOOK: Input: default@t3
1 2
1 2
1 12
+PREHOOK: query: -- 13. null expr in select list
+select null from t3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: -- 13. null expr in select list
+select null from t3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL