You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/25 06:25:30 UTC
svn commit: r1526102 - in /hive/trunk:
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/
serde/src/test/org...
Author: hashutosh
Date: Wed Sep 25 04:25:30 2013
New Revision: 1526102
URL: http://svn.apache.org/r1526102
Log:
HIVE-5329 : Date and timestamp type converts invalid strings to 1970-01-01 (Jason Dere via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q
hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java
hive/trunk/ql/src/test/queries/clientpositive/partition_date.q
hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java Wed Sep 25 04:25:30 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.DateConverter;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
@@ -51,11 +52,13 @@ public class GenericUDFToDate extends Ge
}
try {
argumentOI = (PrimitiveObjectInspector) arguments[0];
+ PrimitiveCategory pc = argumentOI.getPrimitiveCategory();
PrimitiveGrouping pg =
- PrimitiveObjectInspectorUtils.getPrimitiveGrouping(argumentOI.getPrimitiveCategory());
+ PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pc);
switch (pg) {
case DATE_GROUP:
case STRING_GROUP:
+ case VOID_GROUP:
break;
default:
throw new UDFArgumentException(
Modified: hive/trunk/ql/src/test/queries/clientpositive/partition_date.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_date.q?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/partition_date.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/partition_date.q Wed Sep 25 04:25:30 2013
@@ -12,7 +12,7 @@ insert overwrite table partition_date_1
select * from src limit 11;
select distinct dt from partition_date_1;
-select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value;
+select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value;
-- 15
select count(*) from partition_date_1 where dt = date '2000-01-01';
Added: hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q?rev=1526102&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q Wed Sep 25 04:25:30 2013
@@ -0,0 +1,24 @@
+
+-- casting from null should yield null
+select
+ cast(null as tinyint),
+ cast(null as smallint),
+ cast(null as int),
+ cast(null as bigint),
+ cast(null as float),
+ cast(null as double),
+ cast(null as decimal),
+ cast(null as date),
+ cast(null as timestamp),
+ cast(null as string),
+ cast(null as varchar(10)),
+ cast(null as boolean),
+ cast(null as binary)
+from src limit 1;
+
+-- Invalid conversions, should all be null
+select
+ cast('abcd' as date),
+ cast('abcd' as timestamp)
+from src limit 1;
+
Modified: hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out Wed Sep 25 04:25:30 2013
@@ -93,12 +93,12 @@ POSTHOOK: Lineage: partition_date_1 PART
POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
2000-01-01
2013-08-08
-PREHOOK: query: select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value
+PREHOOK: query: select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value
PREHOOK: type: QUERY
PREHOOK: Input: default@partition_date_1
PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2
#### A masked pattern was here ####
-POSTHOOK: query: select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value
+POSTHOOK: query: select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_date_1
POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2
@@ -111,11 +111,11 @@ POSTHOOK: Lineage: partition_date_1 PART
POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-165 val_165 2000-01-01 2 1969-12-31 16:00:00
-238 val_238 2000-01-01 2 1969-12-31 16:00:00
-27 val_27 2000-01-01 2 1969-12-31 16:00:00
-311 val_311 2000-01-01 2 1969-12-31 16:00:00
-86 val_86 2000-01-01 2 1969-12-31 16:00:00
+165 val_165 2000-01-01 2
+238 val_238 2000-01-01 2
+27 val_27 2000-01-01 2
+311 val_311 2000-01-01 2
+86 val_86 2000-01-01 2
PREHOOK: query: -- 15
select count(*) from partition_date_1 where dt = date '2000-01-01'
PREHOOK: type: QUERY
Added: hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out?rev=1526102&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out Wed Sep 25 04:25:30 2013
@@ -0,0 +1,56 @@
+PREHOOK: query: -- casting from null should yield null
+select
+ cast(null as tinyint),
+ cast(null as smallint),
+ cast(null as int),
+ cast(null as bigint),
+ cast(null as float),
+ cast(null as double),
+ cast(null as decimal),
+ cast(null as date),
+ cast(null as timestamp),
+ cast(null as string),
+ cast(null as varchar(10)),
+ cast(null as boolean),
+ cast(null as binary)
+from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- casting from null should yield null
+select
+ cast(null as tinyint),
+ cast(null as smallint),
+ cast(null as int),
+ cast(null as bigint),
+ cast(null as float),
+ cast(null as double),
+ cast(null as decimal),
+ cast(null as date),
+ cast(null as timestamp),
+ cast(null as string),
+ cast(null as varchar(10)),
+ cast(null as boolean),
+ cast(null as binary)
+from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+PREHOOK: query: -- Invalid conversions, should all be null
+select
+ cast('abcd' as date),
+ cast('abcd' as timestamp)
+from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- Invalid conversions, should all be null
+select
+ cast('abcd' as date),
+ cast('abcd' as timestamp)
+from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+NULL NULL
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java Wed Sep 25 04:25:30 2013
@@ -46,11 +46,17 @@ public class JavaDateObjectInspector
}
public Object set(Object o, Date value) {
+ if (value == null) {
+ return null;
+ }
((Date) o).setTime(value.getTime());
return o;
}
public Object set(Object o, DateWritable d) {
+ if (d == null) {
+ return null;
+ }
((Date) o).setTime(d.get().getTime());
return o;
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java Wed Sep 25 04:25:30 2013
@@ -43,6 +43,9 @@ public class JavaTimestampObjectInspecto
}
public Object set(Object o, Timestamp value) {
+ if (value == null) {
+ return null;
+ }
((Timestamp) o).setTime(value.getTime());
return o;
}
@@ -53,6 +56,9 @@ public class JavaTimestampObjectInspecto
}
public Object set(Object o, TimestampWritable tw) {
+ if (tw == null) {
+ return null;
+ }
Timestamp t = (Timestamp) o;
t.setTime(tw.getTimestamp().getTime());
t.setNanos(tw.getTimestamp().getNanos());
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java Wed Sep 25 04:25:30 2013
@@ -1166,7 +1166,8 @@ public final class PrimitiveObjectInspec
* Provide a general grouping for each primitive data type.
*/
public static enum PrimitiveGrouping {
- NUMERIC_GROUP, STRING_GROUP, BOOLEAN_GROUP, DATE_GROUP, BINARY_GROUP, UNKNOWN_GROUP
+ NUMERIC_GROUP, STRING_GROUP, BOOLEAN_GROUP, DATE_GROUP, BINARY_GROUP,
+ VOID_GROUP, UNKNOWN_GROUP
};
/**
@@ -1196,6 +1197,8 @@ public final class PrimitiveObjectInspec
return PrimitiveGrouping.DATE_GROUP;
case BINARY:
return PrimitiveGrouping.BINARY_GROUP;
+ case VOID:
+ return PrimitiveGrouping.VOID_GROUP;
default:
return PrimitiveGrouping.UNKNOWN_GROUP;
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java Wed Sep 25 04:25:30 2013
@@ -46,11 +46,17 @@ public class WritableDateObjectInspector
}
public Object set(Object o, Date d) {
+ if (d == null) {
+ return null;
+ }
((DateWritable) o).set(d);
return o;
}
public Object set(Object o, DateWritable d) {
+ if (d == null) {
+ return null;
+ }
((DateWritable) o).set(d);
return o;
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java Wed Sep 25 04:25:30 2013
@@ -48,11 +48,17 @@ public class WritableTimestampObjectInsp
}
public Object set(Object o, Timestamp t) {
+ if (t == null) {
+ return null;
+ }
((TimestampWritable) o).set(t);
return o;
}
public Object set(Object o, TimestampWritable t) {
+ if (t == null) {
+ return null;
+ }
((TimestampWritable) o).set(t);
return o;
}
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java?rev=1526102&r1=1526101&r2=1526102&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java Wed Sep 25 04:25:30 2013
@@ -39,7 +39,7 @@ public class TestPrimitiveObjectInspecto
assertEquals(PrimitiveGrouping.UNKNOWN_GROUP,
PrimitiveObjectInspectorUtils.getPrimitiveGrouping(PrimitiveCategory.UNKNOWN));
- assertEquals(PrimitiveGrouping.UNKNOWN_GROUP,
+ assertEquals(PrimitiveGrouping.VOID_GROUP,
PrimitiveObjectInspectorUtils.getPrimitiveGrouping(PrimitiveCategory.VOID));
}
}