You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sd...@apache.org on 2011/08/31 02:17:24 UTC
svn commit: r1163455 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/
test/queries/clientnegative/ test/results/clientnegative/
Author: sdong
Date: Wed Aug 31 00:17:24 2011
New Revision: 1163455
URL: http://svn.apache.org/viewvc?rev=1163455&view=rev
Log:
HIVE-2378. Warn user that precision is lost when bigint is implicitly cast to double. (Kevin Wilfong via Siying Dong)
Added:
hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q
hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q
hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out
hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=1163455&r1=1163454&r2=1163455&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Wed Aug 31 00:17:24 2011
@@ -180,6 +180,12 @@ public enum ErrorMsg {
INCOMPATIBLE_SCHEMA("The existing table is not compatible with the import spec. "),
EXIM_FOR_NON_NATIVE("Export/Import cannot be done for a non-native table. "),
INSERT_INTO_BUCKETIZED_TABLE("Bucketized tables do not support INSERT INTO:"),
+ NO_COMPARE_BIGINT_STRING("In strict mode, comparing bigints and strings is not allowed, "
+ + "it may result in a loss of precision. "
+ + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"),
+ NO_COMPARE_BIGINT_DOUBLE("In strict mode, comparing bigints and doubles is not allowed, "
+ + "it may result in a loss of precision. "
+ + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"),
;
private String mesg;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java?rev=1163455&r1=1163454&r2=1163455&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java Wed Aug 31 00:17:24 2011
@@ -22,13 +22,22 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.ErrorMsg;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
@@ -39,9 +48,12 @@ public class ExprNodeGenericFuncDesc ext
private static final long serialVersionUID = 1L;
+ private static final Log LOG = LogFactory
+ .getLog(ExprNodeGenericFuncDesc.class.getName());
+
/**
* In case genericUDF is Serializable, we will serialize the object.
- *
+ *
* In case genericUDF does not implement Serializable, Java will remember the
* class of genericUDF and creates a new instance when deserialized. This is
* exactly what we want.
@@ -138,7 +150,7 @@ public class ExprNodeGenericFuncDesc ext
/**
* Create a exprNodeGenericFuncDesc based on the genericUDFClass and the
* children parameters.
- *
+ *
* @throws UDFArgumentException
*/
public static ExprNodeGenericFuncDesc newInstance(GenericUDF genericUDF,
@@ -148,6 +160,36 @@ public class ExprNodeGenericFuncDesc ext
childrenOIs[i] = children.get(i).getWritableObjectInspector();
}
+ // Check if a bigint is implicitely cast to a double as part of a comparison
+ // Perform the check here instead of in GenericUDFBaseCompare to guarantee it is only run once per operator
+ if (genericUDF instanceof GenericUDFBaseCompare && children.size() == 2) {
+
+ TypeInfo oiTypeInfo0 = children.get(0).getTypeInfo();
+ TypeInfo oiTypeInfo1 = children.get(1).getTypeInfo();
+
+ SessionState ss = SessionState.get();
+ Configuration conf = (ss != null) ? ss.getConf() : new Configuration();
+
+ LogHelper console = new LogHelper(LOG);
+
+ // For now, if a bigint is going to be cast to a double throw an error or warning
+ if ((oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo)) ||
+ (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo))) {
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) {
+ throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_STRING.getMsg());
+ } else {
+ console.printError("WARNING: Comparing a bigint and a string may result in a loss of precision.");
+ }
+ } else if ((oiTypeInfo0.equals(TypeInfoFactory.doubleTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo)) ||
+ (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.doubleTypeInfo))) {
+ if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")) {
+ throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_DOUBLE.getMsg());
+ } else {
+ console.printError("WARNING: Comparing a bigint and a double may result in a loss of precision.");
+ }
+ }
+ }
+
ObjectInspector oi = genericUDF.initialize(childrenOIs);
return new ExprNodeGenericFuncDesc(TypeInfoUtils
.getTypeInfoFromObjectInspector(oi), genericUDF, children);
Added: hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q Wed Aug 31 00:17:24 2011
@@ -0,0 +1,5 @@
+set hive.mapred.mode=strict;
+
+-- This should fail until we fix the issue with precision when casting a bigint to a double
+
+select * from src where cast(1 as bigint) = 1.0 limit 10;
\ No newline at end of file
Added: hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q Wed Aug 31 00:17:24 2011
@@ -0,0 +1,5 @@
+set hive.mapred.mode=strict;
+
+--This should fail until we fix the issue with precision when casting a bigint to a double
+
+select * from src where cast(1 as bigint) = '1' limit 10;
\ No newline at end of file
Added: hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out Wed Aug 31 00:17:24 2011
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 0:-1 Wrong arguments '1.0': In strict mode, comparing bigints and doubles is not allowed, it may result in a loss of precision. If you really want to perform the operation, set hive.mapred.mode=nonstrict
Added: hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out Wed Aug 31 00:17:24 2011
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 0:-1 Wrong arguments ''1'': In strict mode, comparing bigints and strings is not allowed, it may result in a loss of precision. If you really want to perform the operation, set hive.mapred.mode=nonstrict