You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/05/02 22:49:46 UTC
hive git commit: HIVE-19383 : Add ArrayList$SubList kryo serializer
(Ashutosh Chauhan via Prasanth J)
Repository: hive
Updated Branches:
refs/heads/master f44cae409 -> 2c7f9c26e
HIVE-19383 : Add ArrayList$SubList kryo serializer (Ashutosh Chauhan via Prasanth J)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2c7f9c26
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2c7f9c26
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2c7f9c26
Branch: refs/heads/master
Commit: 2c7f9c26e88091ada018e0247c1f6ef11f002606
Parents: f44cae4
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Tue May 1 22:38:08 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed May 2 15:49:34 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/exec/SerializationUtilities.java | 66 ++++++++++++
ql/src/test/queries/clientpositive/kryo.q | 16 +++
.../test/results/clientpositive/llap/kryo.q.out | 104 +++++++++++++++++++
4 files changed, 187 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2c7f9c26/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 6b22afc..13c08de 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -535,6 +535,7 @@ minillaplocal.query.files=\
join_nullsafe.q,\
join_is_not_distinct_from.q,\
join_reordering_no_stats.q,\
+ kryo.q,\
leftsemijoin_mr.q,\
limit_join_transpose.q,\
lineage2.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/2c7f9c26/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
index e1f40cf..66f0a00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
@@ -222,6 +222,8 @@ public class SerializationUtilities {
}
}
+ private static final Object FAKE_REFERENCE = new Object();
+
private static KryoFactory factory = new KryoFactory() {
public Kryo create() {
KryoWithHooks kryo = new KryoWithHooks();
@@ -230,6 +232,7 @@ public class SerializationUtilities {
kryo.register(TimestampTZ.class, new TimestampTZSerializer());
kryo.register(Path.class, new PathSerializer());
kryo.register(Arrays.asList("").getClass(), new ArraysAsListSerializer());
+ kryo.register(new java.util.ArrayList().subList(0,0).getClass(), new ArrayListSubListSerializer());
kryo.register(CopyOnFirstWriteProperties.class, new CopyOnFirstWritePropertiesSerializer());
((Kryo.DefaultInstantiatorStrategy) kryo.getInstantiatorStrategy())
@@ -363,6 +366,69 @@ public class SerializationUtilities {
}
/**
+ * Supports sublists created via {@link ArrayList#subList(int, int)} since java7 (oracle jdk,
+ * represented by <code>java.util.ArrayList$SubList</code>).
+ * This is from kryo-serializers package.
+ */
+ private static class ArrayListSubListSerializer extends com.esotericsoftware.kryo.Serializer<List<?>> {
+
+ private Field _parentField;
+ private Field _parentOffsetField;
+ private Field _sizeField;
+
+ public ArrayListSubListSerializer() {
+ try {
+ final Class<?> clazz = Class.forName("java.util.ArrayList$SubList");
+ _parentField = clazz.getDeclaredField("parent");
+ _parentOffsetField = clazz.getDeclaredField( "parentOffset" );
+ _sizeField = clazz.getDeclaredField( "size" );
+ _parentField.setAccessible( true );
+ _parentOffsetField.setAccessible( true );
+ _sizeField.setAccessible( true );
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public List<?> read(final Kryo kryo, final Input input, final Class<List<?>> clazz) {
+ kryo.reference(FAKE_REFERENCE);
+ final List<?> list = (List<?>) kryo.readClassAndObject(input);
+ final int fromIndex = input.readInt(true);
+ final int toIndex = input.readInt(true);
+ return list.subList(fromIndex, toIndex);
+ }
+
+ @Override
+ public void write(final Kryo kryo, final Output output, final List<?> obj) {
+ try {
+ kryo.writeClassAndObject(output, _parentField.get(obj));
+ final int parentOffset = _parentOffsetField.getInt( obj );
+ final int fromIndex = parentOffset;
+ output.writeInt(fromIndex, true);
+ final int toIndex = fromIndex + _sizeField.getInt( obj );
+ output.writeInt(toIndex, true);
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public List<?> copy(final Kryo kryo, final List<?> original) {
+ try {
+ kryo.reference(FAKE_REFERENCE);
+ final List<?> list = (List<?>) _parentField.get(original);
+ final int parentOffset = _parentOffsetField.getInt( original );
+ final int fromIndex = parentOffset;
+ final int toIndex = fromIndex + _sizeField.getInt( original );
+ return kryo.copy(list).subList(fromIndex, toIndex);
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
* A kryo {@link Serializer} for lists created via {@link Arrays#asList(Object...)}.
* <p>
* Note: This serializer does not support cyclic references, so if one of the objects
http://git-wip-us.apache.org/repos/asf/hive/blob/2c7f9c26/ql/src/test/queries/clientpositive/kryo.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/kryo.q b/ql/src/test/queries/clientpositive/kryo.q
new file mode 100644
index 0000000..eba10c1
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/kryo.q
@@ -0,0 +1,16 @@
+CREATE TABLE `DateTime` (
+ ID INT,
+ `Time` STRING,
+ Date_ STRING,
+ `DateTime` STRING,
+ Random_ DOUBLE,
+ Diagonal DOUBLE,
+ Curved DOUBLE
+ )
+STORED AS orc;
+
+explain SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5)
THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTi
me_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN
(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) ;
+
+SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CA
SE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR `DateTime_`.`id
` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 5) THEN (CASE
WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) ;
+
+drop table DateTime;
http://git-wip-us.apache.org/repos/asf/hive/blob/2c7f9c26/ql/src/test/results/clientpositive/llap/kryo.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/kryo.q.out b/ql/src/test/results/clientpositive/llap/kryo.q.out
new file mode 100644
index 0000000..1463dfb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/kryo.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: CREATE TABLE `DateTime` (
+ ID INT,
+ `Time` STRING,
+ Date_ STRING,
+ `DateTime` STRING,
+ Random_ DOUBLE,
+ Diagonal DOUBLE,
+ Curved DOUBLE
+ )
+STORED AS orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@DateTime
+POSTHOOK: query: CREATE TABLE `DateTime` (
+ ID INT,
+ `Time` STRING,
+ Date_ STRING,
+ `DateTime` STRING,
+ Random_ DOUBLE,
+ Diagonal DOUBLE,
+ Curved DOUBLE
+ )
+STORED AS orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@DateTime
+PREHOOK: query: explain SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateT
ime_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id
` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_
`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`Date
Time_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`i
d` = 4 OR `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime
_`.`id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: datetime_
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((id = 1) or (id = 2) or (id = 3) or (id = 4) or (id = 5) or (id = 6)) (type: boolean)
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: if(((id = 1) or (id = 2)), if((id = 1), date_, date_), if((id = 3), CASE WHEN ((date_ is null or to_date(datetime) is null)) THEN (null) WHEN ((CAST( date_ AS DATE) > to_date(datetime))) THEN (date_) ELSE (to_date(datetime)) END, null)) (type: string), id (type: int), CASE WHEN ((id = 6)) THEN (CASE WHEN ((concat(date_, ' 00:00:00') is null or datetime is null)) THEN (null) WHEN ((concat(date_, ' 00:00:00') > datetime)) THEN (concat(date_, ' 00:00:00')) ELSE (datetime) END) WHEN ((id = 5)) THEN (CASE WHEN ((date_ is null or datetime is null)) THEN (null) WHEN ((date_ > datetime)) THEN (date_) ELSE (datetime) END) WHEN ((id = 3)) THEN (concat(date_, ' 00:00:00')) WHEN ((id = 4)) THEN (concat(date_, ' 00:00:00')) WHEN ((id = 1)) THEN (date_) WHEN ((id = 2)) THEN (date_) ELSE (null) END (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string), _col1 (type: int), _col2 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string)
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`i
d` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 OR
`DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` =
5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@datetime
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)) AS `none_calculation2_ok`, `DateTime_`.`id` AS `none_id_ok`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`
id` = 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END) AS `none_mixed_type_calcs_ok` FROM `DateTime` `DateTime_` WHERE (`DateTime_`.`id` = 1 OR `DateTime_`.`id` = 2 OR `DateTime_`.`id` = 3 OR `DateTime_`.`id` = 4 O
R `DateTime_`.`id` = 5 OR `DateTime_`.`id` = 6) GROUP BY IF(((`DateTime_`.`id` = 1) OR (`DateTime_`.`id` = 2)),IF((`DateTime_`.`id` = 1),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE NULL END),(CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN TO_DATE(`DateTime_`.`datetime`) ELSE TO_DATE(`DateTime_`.`time`) END)),IF((`DateTime_`.`id` = 3),(CASE WHEN (`DateTime_`.`date_` IS NULL) OR (TO_DATE(`DateTime_`.`datetime`) IS NULL) THEN NULL WHEN `DateTime_`.`date_` > TO_DATE(`DateTime_`.`datetime`) THEN `DateTime_`.`date_` ELSE TO_DATE(`DateTime_`.`datetime`) END),NULL)), `DateTime_`.`id`, (CASE WHEN (`DateTime_`.`id` = 6) THEN (CASE WHEN (CONCAT(`DateTime_`.`date_`, ' 00:00:00') IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') > `DateTime_`.`datetime` THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id`
= 5) THEN (CASE WHEN (`DateTime_`.`date_` IS NULL) OR (`DateTime_`.`datetime` IS NULL) THEN NULL WHEN `DateTime_`.`date_` > `DateTime_`.`datetime` THEN `DateTime_`.`date_` ELSE `DateTime_`.`datetime` END) WHEN (`DateTime_`.`id` = 3) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 4) THEN (CASE WHEN true THEN CONCAT(`DateTime_`.`date_`, ' 00:00:00') WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) WHEN (`DateTime_`.`id` = 1) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE NULL END) WHEN (`DateTime_`.`id` = 2) THEN (CASE WHEN true THEN `DateTime_`.`date_` WHEN NOT true THEN `DateTime_`.`datetime` ELSE `DateTime_`.`time` END) ELSE NULL END)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@datetime
+#### A masked pattern was here ####
+PREHOOK: query: drop table DateTime
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@datetime
+PREHOOK: Output: default@datetime
+POSTHOOK: query: drop table DateTime
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@datetime
+POSTHOOK: Output: default@datetime