You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/10/23 17:25:08 UTC
svn commit: r1633841 - in /hive/branches/branch-0.14/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/
java/org/apache/hadoop/hive/ql/plan/ test/queries/clientpositive/
test/results/clientpositive/
Author: hashutosh
Date: Thu Oct 23 15:25:07 2014
New Revision: 1633841
URL: http://svn.apache.org/r1633841
Log:
HIVE-8555 : Too many casts results in loss of original string representation for constant (Ashutosh Chauhan via Sergey Shelukhin)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/constprog_type.q
hive/branches/branch-0.14/ql/src/test/results/clientpositive/constprog_type.q.out
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1633841&r1=1633840&r2=1633841&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Thu Oct 23 15:25:07 2014
@@ -1711,13 +1711,13 @@ public final class FunctionRegistry {
if (!(desc instanceof ExprNodeGenericFuncDesc)) {
return false;
}
- GenericUDF genericUDF = ((ExprNodeGenericFuncDesc)desc).getGenericUDF();
- Class udfClass;
- if (genericUDF instanceof GenericUDFBridge) {
- udfClass = ((GenericUDFBridge)genericUDF).getUdfClass();
- } else {
- udfClass = genericUDF.getClass();
- }
+ return isOpCast(((ExprNodeGenericFuncDesc)desc).getGenericUDF());
+ }
+
+ public static boolean isOpCast(GenericUDF genericUDF) {
+ Class udfClass = (genericUDF instanceof GenericUDFBridge) ?
+ ((GenericUDFBridge)genericUDF).getUdfClass() : genericUDF.getClass();
+
return udfClass == UDFToBoolean.class || udfClass == UDFToByte.class ||
udfClass == UDFToDouble.class || udfClass == UDFToFloat.class ||
udfClass == UDFToInteger.class || udfClass == UDFToLong.class ||
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java?rev=1633841&r1=1633840&r2=1633841&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java Thu Oct 23 15:25:07 2014
@@ -166,6 +166,10 @@ public final class ConstantPropagateProc
}
LOG.debug("Casting " + desc + " to type " + ti);
ExprNodeConstantDesc c = (ExprNodeConstantDesc) desc;
+ if (null != c.getFoldedFromVal() && priti.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
+ // avoid double casting to preserve original string representation of constant.
+ return new ExprNodeConstantDesc(c.getFoldedFromVal());
+ }
ObjectInspector origOI =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(desc.getTypeInfo());
ObjectInspector oi =
@@ -550,7 +554,12 @@ public final class ConstantPropagateProc
LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable.");
return null;
}
- return new ExprNodeConstantDesc(o);
+ String constStr = null;
+ if(arguments.length == 1 && FunctionRegistry.isOpCast(udf)) {
+ // remember original string representation of constant.
+ constStr = arguments[0].get().toString();
+ }
+ return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr);
} catch (HiveException e) {
LOG.error("Evaluation function " + udf.getClass()
+ " failed in Constant Propagatation Optimizer.");
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1633841&r1=1633840&r2=1633841&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original)
+++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Thu Oct 23 15:25:07 2014
@@ -40,6 +40,17 @@ public class ExprNodeConstantDesc extend
// If this constant was created while doing constant folding, foldedFromCol holds the name of
// original column from which it was folded.
private transient String foldedFromCol;
+ // string representation of folding constant.
+ private transient String foldedFromVal;
+
+ public ExprNodeConstantDesc setFoldedFromVal(String foldedFromVal) {
+ this.foldedFromVal = foldedFromVal;
+ return this;
+ }
+
+ public String getFoldedFromVal() {
+ return foldedFromVal;
+ }
public String getFoldedFromCol() {
return foldedFromCol;
Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/constprog_type.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/constprog_type.q?rev=1633841&r1=1633840&r2=1633841&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/constprog_type.q (original)
+++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/constprog_type.q Thu Oct 23 15:25:07 2014
@@ -12,3 +12,28 @@ SELECT cast('2013-11-17' as date), cast(
FROM src tablesample (1 rows);
SELECT * FROM dest1;
+
+SELECT key, value FROM src WHERE key = cast(86 as double);
+
+CREATE TABLE primitives1 (
+ id INT ,
+ bool_col BOOLEAN ,
+ tinyint_col TINYINT ,
+ smallint_col SMALLINT ,
+ int_col INT ,
+ bigint_col BIGINT ,
+ float_col FLOAT ,
+ double_col DOUBLE ,
+ date_string_col STRING ,
+ string_col STRING ,
+ timestamp_col TIMESTAMP )
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY ','
+ ESCAPED BY '\\'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt'
+OVERWRITE INTO TABLE primitives1 ;
+
+
+select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col = cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double) and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and double_col = cast (0.0 as float);
Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/constprog_type.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/constprog_type.q.out?rev=1633841&r1=1633840&r2=1633841&view=diff
==============================================================================
--- hive/branches/branch-0.14/ql/src/test/results/clientpositive/constprog_type.q.out (original)
+++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/constprog_type.q.out Thu Oct 23 15:25:07 2014
@@ -122,3 +122,69 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
2013-11-17 2011-04-29 20:46:56.4485
+PREHOOK: query: SELECT key, value FROM src WHERE key = cast(86 as double)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT key, value FROM src WHERE key = cast(86 as double)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+86 val_86
+PREHOOK: query: CREATE TABLE primitives1 (
+ id INT ,
+ bool_col BOOLEAN ,
+ tinyint_col TINYINT ,
+ smallint_col SMALLINT ,
+ int_col INT ,
+ bigint_col BIGINT ,
+ float_col FLOAT ,
+ double_col DOUBLE ,
+ date_string_col STRING ,
+ string_col STRING ,
+ timestamp_col TIMESTAMP )
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY ','
+ ESCAPED BY '\\'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@primitives1
+POSTHOOK: query: CREATE TABLE primitives1 (
+ id INT ,
+ bool_col BOOLEAN ,
+ tinyint_col TINYINT ,
+ smallint_col SMALLINT ,
+ int_col INT ,
+ bigint_col BIGINT ,
+ float_col FLOAT ,
+ double_col DOUBLE ,
+ date_string_col STRING ,
+ string_col STRING ,
+ timestamp_col TIMESTAMP )
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY ','
+ ESCAPED BY '\\'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@primitives1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt'
+OVERWRITE INTO TABLE primitives1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@primitives1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt'
+OVERWRITE INTO TABLE primitives1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@primitives1
+PREHOOK: query: select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col = cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double) and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and double_col = cast (0.0 as float)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@primitives1
+#### A masked pattern was here ####
+POSTHOOK: query: select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col = cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double) and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and double_col = cast (0.0 as float)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@primitives1
+#### A masked pattern was here ####
+0 true 0 0 0 0 0.0 0.0