You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2019/09/27 06:08:23 UTC
[hive] branch master updated: HIVE-22197 : Common Merge join
throwing class cast exception. (Mahesh Kumar Behera reviewed by Vineet
Garg)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 33ccc9b HIVE-22197 : Common Merge join throwing class cast exception. (Mahesh Kumar Behera reviewed by Vineet Garg)
33ccc9b is described below
commit 33ccc9bef82f7acb7fa7cb7ab33a654b1790fd45
Author: Mahesh Kumar Behera <ma...@apache.org>
AuthorDate: Fri Sep 27 11:36:17 2019 +0530
HIVE-22197 : Common Merge join throwing class cast exception. (Mahesh Kumar Behera reviewed by Vineet Garg)
---
.../hadoop/hive/ql/exec/DummyStoreOperator.java | 4 +-
.../clientpositive/merge_test_dummy_operator.q | 120 ++++++
.../clientpositive/merge_test_dummy_operator.q.out | 419 +++++++++++++++++++++
.../tez/merge_test_dummy_operator.q.out | 419 +++++++++++++++++++++
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 1 +
5 files changed, 960 insertions(+), 3 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
index 2c63001..7af70aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DummyStoreOperator.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.ql.plan.DummyStoreDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
/**
* For SortMerge joins, this is a dummy operator, which stores the row for the
@@ -99,8 +98,7 @@ public class DummyStoreOperator extends Operator<DummyStoreDesc> implements Seri
@Override
public void process(Object row, int tag) throws HiveException {
// Store the row. See comments above for why we need a new copy of the row.
- result.o = ObjectInspectorUtils.copyToStandardObject(row, inputObjInspectors[0],
- ObjectInspectorCopyOption.WRITABLE);
+ result.o = ObjectInspectorUtils.copyToStandardObject(row, inputObjInspectors[0]);
}
@Override
diff --git a/ql/src/test/queries/clientpositive/merge_test_dummy_operator.q b/ql/src/test/queries/clientpositive/merge_test_dummy_operator.q
new file mode 100644
index 0000000..f11e22c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/merge_test_dummy_operator.q
@@ -0,0 +1,120 @@
+--! qt:dataset:src
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.auto.convert.join=false;
+
+create table common_join_table (id string,
+ col1 string,
+ date_created date,
+ col2 string,
+ col3 string,
+ time_stamp timestamp,
+ col4 date,
+ col4key bigint,
+ col5 date,
+ col6 string,
+ col7 string,
+ col8 smallint);
+
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null);
+
+WITH temp_tbl_1 AS (
+SELECT col7
+ ,col4KEY
+ ,COUNT(*) AS temp_result_1
+ FROM common_join_table
+ GROUP BY col7, col4KEY
+),
+
+temp_tbl_2 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_1
+ ,ROW_NUMBER() OVER(PARTITION BY col7 ORDER BY col4KEY ASC) AS temp_result_2
+ FROM temp_tbl_1
+),
+
+temp_tbl_3 AS (
+SELECT col7
+ ,MIN(col4KEY) AS START_DATE
+ ,MAX(col4KEY) AS END_DATE
+ FROM temp_tbl_2
+ GROUP BY col7
+),
+
+
+temp_tbl_4 AS (
+SELECT D1.col7
+ ,D1.col4KEY
+ ,D1.temp_result_2
+ ,D1.temp_result_1
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D1.col4KEY
+ WHEN D1.col4KEY = M.END_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_3
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D2.col4KEY
+ WHEN D1.col4KEY = M.START_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_4
+ FROM temp_tbl_2 D1
+ INNER JOIN temp_tbl_3 M
+ ON D1.col7 = M.col7
+ LEFT JOIN temp_tbl_2 D2
+ ON D1.col7 = D2.col7
+ AND D1.temp_result_2 = D2.temp_result_2+1
+),
+
+temp_tbl_5 AS (
+SELECT S1.col7
+ ,S1.col4KEY
+ ,S1.temp_result_2
+ ,S1.temp_result_1
+ ,CASE WHEN S1.col4KEY >= S2.temp_result_4
+ AND S1.col4KEY <= S3.temp_result_3
+ THEN 1 ELSE 0 END AS temp_result_5
+ FROM temp_tbl_4 S1
+ LEFT JOIN temp_tbl_4 S2
+ ON S1.col7 = S2.col7
+ AND S2.temp_result_4 != 0
+ LEFT JOIN temp_tbl_4 S3
+ ON S1.col7 = S3.col7
+ AND S3.temp_result_3 != 0
+),
+
+temp_tbl_6 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+ ,SUM(temp_result_5) AS temp_result_5
+ FROM temp_tbl_5
+ GROUP BY col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+),
+
+temp_tbl_7 AS (
+SELECT col7
+ ,SUM(temp_result_2) AS temp_result_6
+ ,SUM(temp_result_1) AS temp_result_1
+ FROM temp_tbl_6
+ GROUP BY col7
+)
+
+SELECT S.*
+ FROM temp_tbl_6 S
+ INNER JOIN
+ temp_tbl_7 F
+ ON S.col7 = F.col7
+ --WHERE F.temp_result_6 < 40
+ --AND F.temp_result_1 < 200
+;
+
+drop table common_join_table;
+
diff --git a/ql/src/test/results/clientpositive/merge_test_dummy_operator.q.out b/ql/src/test/results/clientpositive/merge_test_dummy_operator.q.out
new file mode 100644
index 0000000..31d4ae1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/merge_test_dummy_operator.q.out
@@ -0,0 +1,419 @@
+PREHOOK: query: create table common_join_table (id string,
+ col1 string,
+ date_created date,
+ col2 string,
+ col3 string,
+ time_stamp timestamp,
+ col4 date,
+ col4key bigint,
+ col5 date,
+ col6 string,
+ col7 string,
+ col8 smallint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: create table common_join_table (id string,
+ col1 string,
+ date_created date,
+ col2 string,
+ col3 string,
+ time_stamp timestamp,
+ col4 date,
+ col4key bigint,
+ col5 date,
+ col6 string,
+ col7 string,
+ col8 smallint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@common_join_table
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: WITH temp_tbl_1 AS (
+SELECT col7
+ ,col4KEY
+ ,COUNT(*) AS temp_result_1
+ FROM common_join_table
+ GROUP BY col7, col4KEY
+),
+
+temp_tbl_2 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_1
+ ,ROW_NUMBER() OVER(PARTITION BY col7 ORDER BY col4KEY ASC) AS temp_result_2
+ FROM temp_tbl_1
+),
+
+temp_tbl_3 AS (
+SELECT col7
+ ,MIN(col4KEY) AS START_DATE
+ ,MAX(col4KEY) AS END_DATE
+ FROM temp_tbl_2
+ GROUP BY col7
+),
+
+
+temp_tbl_4 AS (
+SELECT D1.col7
+ ,D1.col4KEY
+ ,D1.temp_result_2
+ ,D1.temp_result_1
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D1.col4KEY
+ WHEN D1.col4KEY = M.END_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_3
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D2.col4KEY
+ WHEN D1.col4KEY = M.START_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_4
+ FROM temp_tbl_2 D1
+ INNER JOIN temp_tbl_3 M
+ ON D1.col7 = M.col7
+ LEFT JOIN temp_tbl_2 D2
+ ON D1.col7 = D2.col7
+ AND D1.temp_result_2 = D2.temp_result_2+1
+),
+
+temp_tbl_5 AS (
+SELECT S1.col7
+ ,S1.col4KEY
+ ,S1.temp_result_2
+ ,S1.temp_result_1
+ ,CASE WHEN S1.col4KEY >= S2.temp_result_4
+ AND S1.col4KEY <= S3.temp_result_3
+ THEN 1 ELSE 0 END AS temp_result_5
+ FROM temp_tbl_4 S1
+ LEFT JOIN temp_tbl_4 S2
+ ON S1.col7 = S2.col7
+ AND S2.temp_result_4 != 0
+ LEFT JOIN temp_tbl_4 S3
+ ON S1.col7 = S3.col7
+ AND S3.temp_result_3 != 0
+),
+
+temp_tbl_6 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+ ,SUM(temp_result_5) AS temp_result_5
+ FROM temp_tbl_5
+ GROUP BY col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+),
+
+temp_tbl_7 AS (
+SELECT col7
+ ,SUM(temp_result_2) AS temp_result_6
+ ,SUM(temp_result_1) AS temp_result_1
+ FROM temp_tbl_6
+ GROUP BY col7
+)
+
+SELECT S.*
+ FROM temp_tbl_6 S
+ INNER JOIN
+ temp_tbl_7 F
+ ON S.col7 = F.col7
+ --WHERE F.temp_result_6 < 40
+ --AND F.temp_result_1 < 200
+PREHOOK: type: QUERY
+PREHOOK: Input: default@common_join_table
+#### A masked pattern was here ####
+POSTHOOK: query: WITH temp_tbl_1 AS (
+SELECT col7
+ ,col4KEY
+ ,COUNT(*) AS temp_result_1
+ FROM common_join_table
+ GROUP BY col7, col4KEY
+),
+
+temp_tbl_2 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_1
+ ,ROW_NUMBER() OVER(PARTITION BY col7 ORDER BY col4KEY ASC) AS temp_result_2
+ FROM temp_tbl_1
+),
+
+temp_tbl_3 AS (
+SELECT col7
+ ,MIN(col4KEY) AS START_DATE
+ ,MAX(col4KEY) AS END_DATE
+ FROM temp_tbl_2
+ GROUP BY col7
+),
+
+
+temp_tbl_4 AS (
+SELECT D1.col7
+ ,D1.col4KEY
+ ,D1.temp_result_2
+ ,D1.temp_result_1
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D1.col4KEY
+ WHEN D1.col4KEY = M.END_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_3
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D2.col4KEY
+ WHEN D1.col4KEY = M.START_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_4
+ FROM temp_tbl_2 D1
+ INNER JOIN temp_tbl_3 M
+ ON D1.col7 = M.col7
+ LEFT JOIN temp_tbl_2 D2
+ ON D1.col7 = D2.col7
+ AND D1.temp_result_2 = D2.temp_result_2+1
+),
+
+temp_tbl_5 AS (
+SELECT S1.col7
+ ,S1.col4KEY
+ ,S1.temp_result_2
+ ,S1.temp_result_1
+ ,CASE WHEN S1.col4KEY >= S2.temp_result_4
+ AND S1.col4KEY <= S3.temp_result_3
+ THEN 1 ELSE 0 END AS temp_result_5
+ FROM temp_tbl_4 S1
+ LEFT JOIN temp_tbl_4 S2
+ ON S1.col7 = S2.col7
+ AND S2.temp_result_4 != 0
+ LEFT JOIN temp_tbl_4 S3
+ ON S1.col7 = S3.col7
+ AND S3.temp_result_3 != 0
+),
+
+temp_tbl_6 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+ ,SUM(temp_result_5) AS temp_result_5
+ FROM temp_tbl_5
+ GROUP BY col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+),
+
+temp_tbl_7 AS (
+SELECT col7
+ ,SUM(temp_result_2) AS temp_result_6
+ ,SUM(temp_result_1) AS temp_result_1
+ FROM temp_tbl_6
+ GROUP BY col7
+)
+
+SELECT S.*
+ FROM temp_tbl_6 S
+ INNER JOIN
+ temp_tbl_7 F
+ ON S.col7 = F.col7
+ --WHERE F.temp_result_6 < 40
+ --AND F.temp_result_1 < 200
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@common_join_table
+#### A masked pattern was here ####
+id 109515 1 10 1
+PREHOOK: query: drop table common_join_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@common_join_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: drop table common_join_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@common_join_table
+POSTHOOK: Output: default@common_join_table
diff --git a/ql/src/test/results/clientpositive/tez/merge_test_dummy_operator.q.out b/ql/src/test/results/clientpositive/tez/merge_test_dummy_operator.q.out
new file mode 100644
index 0000000..22c64a1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/merge_test_dummy_operator.q.out
@@ -0,0 +1,419 @@
+PREHOOK: query: create table common_join_table (id string,
+ col1 string,
+ date_created date,
+ col2 string,
+ col3 string,
+ time_stamp timestamp,
+ col4 date,
+ col4key bigint,
+ col5 date,
+ col6 string,
+ col7 string,
+ col8 smallint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: create table common_join_table (id string,
+ col1 string,
+ date_created date,
+ col2 string,
+ col3 string,
+ time_stamp timestamp,
+ col4 date,
+ col4key bigint,
+ col5 date,
+ col6 string,
+ col7 string,
+ col8 smallint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@common_join_table
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: insert into common_join_table values ('id', '109515', null, 'test', 'test', '2018-01-10 15:03:55.0', '2018-01-10', 109515, null, '45045501', 'id', null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@common_join_table
+POSTHOOK: Lineage: common_join_table.col1 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col2 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col3 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col4key SCRIPT []
+POSTHOOK: Lineage: common_join_table.col5 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.col6 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col7 SCRIPT []
+POSTHOOK: Lineage: common_join_table.col8 EXPRESSION []
+POSTHOOK: Lineage: common_join_table.date_created EXPRESSION []
+POSTHOOK: Lineage: common_join_table.id SCRIPT []
+POSTHOOK: Lineage: common_join_table.time_stamp SCRIPT []
+PREHOOK: query: WITH temp_tbl_1 AS (
+SELECT col7
+ ,col4KEY
+ ,COUNT(*) AS temp_result_1
+ FROM common_join_table
+ GROUP BY col7, col4KEY
+),
+
+temp_tbl_2 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_1
+ ,ROW_NUMBER() OVER(PARTITION BY col7 ORDER BY col4KEY ASC) AS temp_result_2
+ FROM temp_tbl_1
+),
+
+temp_tbl_3 AS (
+SELECT col7
+ ,MIN(col4KEY) AS START_DATE
+ ,MAX(col4KEY) AS END_DATE
+ FROM temp_tbl_2
+ GROUP BY col7
+),
+
+
+temp_tbl_4 AS (
+SELECT D1.col7
+ ,D1.col4KEY
+ ,D1.temp_result_2
+ ,D1.temp_result_1
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D1.col4KEY
+ WHEN D1.col4KEY = M.END_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_3
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D2.col4KEY
+ WHEN D1.col4KEY = M.START_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_4
+ FROM temp_tbl_2 D1
+ INNER JOIN temp_tbl_3 M
+ ON D1.col7 = M.col7
+ LEFT JOIN temp_tbl_2 D2
+ ON D1.col7 = D2.col7
+ AND D1.temp_result_2 = D2.temp_result_2+1
+),
+
+temp_tbl_5 AS (
+SELECT S1.col7
+ ,S1.col4KEY
+ ,S1.temp_result_2
+ ,S1.temp_result_1
+ ,CASE WHEN S1.col4KEY >= S2.temp_result_4
+ AND S1.col4KEY <= S3.temp_result_3
+ THEN 1 ELSE 0 END AS temp_result_5
+ FROM temp_tbl_4 S1
+ LEFT JOIN temp_tbl_4 S2
+ ON S1.col7 = S2.col7
+ AND S2.temp_result_4 != 0
+ LEFT JOIN temp_tbl_4 S3
+ ON S1.col7 = S3.col7
+ AND S3.temp_result_3 != 0
+),
+
+temp_tbl_6 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+ ,SUM(temp_result_5) AS temp_result_5
+ FROM temp_tbl_5
+ GROUP BY col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+),
+
+temp_tbl_7 AS (
+SELECT col7
+ ,SUM(temp_result_2) AS temp_result_6
+ ,SUM(temp_result_1) AS temp_result_1
+ FROM temp_tbl_6
+ GROUP BY col7
+)
+
+SELECT S.*
+ FROM temp_tbl_6 S
+ INNER JOIN
+ temp_tbl_7 F
+ ON S.col7 = F.col7
+ --WHERE F.temp_result_6 < 40
+ --AND F.temp_result_1 < 200
+PREHOOK: type: QUERY
+PREHOOK: Input: default@common_join_table
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: WITH temp_tbl_1 AS (
+SELECT col7
+ ,col4KEY
+ ,COUNT(*) AS temp_result_1
+ FROM common_join_table
+ GROUP BY col7, col4KEY
+),
+
+temp_tbl_2 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_1
+ ,ROW_NUMBER() OVER(PARTITION BY col7 ORDER BY col4KEY ASC) AS temp_result_2
+ FROM temp_tbl_1
+),
+
+temp_tbl_3 AS (
+SELECT col7
+ ,MIN(col4KEY) AS START_DATE
+ ,MAX(col4KEY) AS END_DATE
+ FROM temp_tbl_2
+ GROUP BY col7
+),
+
+
+temp_tbl_4 AS (
+SELECT D1.col7
+ ,D1.col4KEY
+ ,D1.temp_result_2
+ ,D1.temp_result_1
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D1.col4KEY
+ WHEN D1.col4KEY = M.END_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_3
+ ,CASE WHEN D2.col4KEY-D1.col4KEY > 30 THEN D2.col4KEY
+ WHEN D1.col4KEY = M.START_DATE THEN D1.col4KEY ELSE 0 END AS temp_result_4
+ FROM temp_tbl_2 D1
+ INNER JOIN temp_tbl_3 M
+ ON D1.col7 = M.col7
+ LEFT JOIN temp_tbl_2 D2
+ ON D1.col7 = D2.col7
+ AND D1.temp_result_2 = D2.temp_result_2+1
+),
+
+temp_tbl_5 AS (
+SELECT S1.col7
+ ,S1.col4KEY
+ ,S1.temp_result_2
+ ,S1.temp_result_1
+ ,CASE WHEN S1.col4KEY >= S2.temp_result_4
+ AND S1.col4KEY <= S3.temp_result_3
+ THEN 1 ELSE 0 END AS temp_result_5
+ FROM temp_tbl_4 S1
+ LEFT JOIN temp_tbl_4 S2
+ ON S1.col7 = S2.col7
+ AND S2.temp_result_4 != 0
+ LEFT JOIN temp_tbl_4 S3
+ ON S1.col7 = S3.col7
+ AND S3.temp_result_3 != 0
+),
+
+temp_tbl_6 AS (
+SELECT col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+ ,SUM(temp_result_5) AS temp_result_5
+ FROM temp_tbl_5
+ GROUP BY col7
+ ,col4KEY
+ ,temp_result_2
+ ,temp_result_1
+),
+
+temp_tbl_7 AS (
+SELECT col7
+ ,SUM(temp_result_2) AS temp_result_6
+ ,SUM(temp_result_1) AS temp_result_1
+ FROM temp_tbl_6
+ GROUP BY col7
+)
+
+SELECT S.*
+ FROM temp_tbl_6 S
+ INNER JOIN
+ temp_tbl_7 F
+ ON S.col7 = F.col7
+ --WHERE F.temp_result_6 < 40
+ --AND F.temp_result_1 < 200
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@common_join_table
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+id 109515 1 10 1
+PREHOOK: query: drop table common_join_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@common_join_table
+PREHOOK: Output: default@common_join_table
+POSTHOOK: query: drop table common_join_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@common_join_table
+POSTHOOK: Output: default@common_join_table
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 9a1e590..3569499 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -416,6 +416,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24);
conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
+ conf.setInt(TezConfiguration.TEZ_COUNTERS_MAX, 1024);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");