You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2017/06/28 19:11:34 UTC

[1/6] hive git commit: HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Repository: hive
Updated Branches:
  refs/heads/master 99c0cafe5 -> 3192d06d6


http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
index 03eb4b6..b7ef7a0 100644
--- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
+++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
@@ -62,22 +62,22 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (key > '9')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((key > '9') and value is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string), _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: string), _col1 (type: string)
                     sort order: ++
                     Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
@@ -86,10 +86,10 @@ STAGE PLANS:
             0 _col0 (type: string), _col1 (type: string)
             1 _col0 (type: string), _col1 (type: string)
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
-            Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
             table:
                 input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -129,22 +129,22 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (key > '9')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((key > '9') and value is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string), _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: string), _col1 (type: string)
                     sort order: ++
                     Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
@@ -210,22 +210,22 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (key > '9')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((key > '9') and value is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string), _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: string), _col1 (type: string)
                     sort order: ++
                     Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
@@ -284,20 +284,20 @@ STAGE PLANS:
             alias: part
             Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: (p_mfgr = p_mfgr) (type: boolean)
-              Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+              predicate: p_mfgr is not null (type: boolean)
+              Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: p_mfgr (type: string), p_size (type: int)
                 sort order: ++
                 Map-reduce partition columns: p_mfgr (type: string)
-                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                 TopN Hash Memory Usage: 0.1
                 value expressions: p_name (type: string)
       Reduce Operator Tree:
         Select Operator
           expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
           outputColumnNames: _col1, _col2, _col5
-          Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
           PTF Operator
             Function definitions:
                 Input definition
@@ -318,20 +318,20 @@ STAGE PLANS:
                         window function: GenericUDAFRankEvaluator
                         window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                         isPivotResult: true
-            Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
               predicate: (rank_window_0 <= 2) (type: boolean)
-              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: _col1 (type: string), _col2 (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   aggregations: count(), count(_col0)
                   keys: _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
                     table:
@@ -347,7 +347,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string)
               sort order: +
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: bigint), _col2 (type: bigint)
       Reduce Operator Tree:
         Group By Operator
@@ -355,7 +355,7 @@ STAGE PLANS:
           keys: KEY._col0 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2
-          Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
             table:
@@ -384,7 +384,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string)
               sort order: +
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: bigint), _col2 (type: bigint)
       Reduce Operator Tree:
         Join Operator
@@ -417,7 +417,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col2 (type: boolean)
       Reduce Operator Tree:
         Join Operator
@@ -450,20 +450,20 @@ STAGE PLANS:
             alias: part
             Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: (p_mfgr = p_mfgr) (type: boolean)
-              Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+              predicate: p_mfgr is not null (type: boolean)
+              Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
                 key expressions: p_mfgr (type: string), p_size (type: int)
                 sort order: ++
                 Map-reduce partition columns: p_mfgr (type: string)
-                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                 TopN Hash Memory Usage: 0.1
                 value expressions: p_name (type: string)
       Reduce Operator Tree:
         Select Operator
           expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
           outputColumnNames: _col1, _col2, _col5
-          Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
           PTF Operator
             Function definitions:
                 Input definition
@@ -484,19 +484,19 @@ STAGE PLANS:
                         window function: GenericUDAFRankEvaluator
                         window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                         isPivotResult: true
-            Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
               predicate: (rank_window_0 <= 2) (type: boolean)
-              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: _col1 (type: string), _col2 (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string), _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
                     table:
@@ -512,20 +512,20 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
             predicate: _col0 is not null (type: boolean)
-            Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
               outputColumnNames: _col0, _col1, _col2
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
                 table:

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
index 0ff11df..e20bdf5 100644
--- a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
@@ -361,12 +361,12 @@ STAGE PLANS:
             alias: lineitem
             Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-              Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: l_orderkey (type: int), l_linenumber (type: int)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   Group By Vectorization:
                       groupByMode: HASH
@@ -377,7 +377,7 @@ STAGE PLANS:
                   keys: _col0 (type: int), _col1 (type: int)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                   HashTable Sink Operator
                     keys:
                       0 _col1 (type: int), _col4 (type: int)


[4/6] hive git commit: HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/perf/query16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query16.q.out b/ql/src/test/results/clientpositive/perf/query16.q.out
index 239f2b7..662bc97 100644
--- a/ql/src/test/results/clientpositive/perf/query16.q.out
+++ b/ql/src/test/results/clientpositive/perf/query16.q.out
@@ -1,4 +1,3 @@
-Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product
 PREHOOK: query: explain
 select  
    count(distinct cs_order_number) as `order count`
@@ -62,180 +61,132 @@ POSTHOOK: type: QUERY
 Plan optimized by CBO.
 
 Vertex dependency in root stage
-Reducer 14 <- Map 13 (SIMPLE_EDGE)
-Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
-Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE)
-Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
-Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 17 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
-Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+Reducer 13 <- Map 12 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
+Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
 
 Stage-0
   Fetch Operator
     limit:-1
     Stage-1
-      Reducer 9
-      File Output Operator [FS_74]
-        Limit [LIM_72] (rows=1 width=344)
+      Reducer 8
+      File Output Operator [FS_50]
+        Limit [LIM_48] (rows=1 width=344)
           Number of rows:100
-          Select Operator [SEL_71] (rows=1 width=344)
+          Select Operator [SEL_47] (rows=1 width=344)
             Output:["_col0","_col1","_col2"]
-          <-Reducer 8 [SIMPLE_EDGE]
-            SHUFFLE [RS_70]
-              Select Operator [SEL_69] (rows=1 width=344)
+          <-Reducer 7 [SIMPLE_EDGE]
+            SHUFFLE [RS_46]
+              Select Operator [SEL_45] (rows=1 width=344)
                 Output:["_col1","_col2","_col3"]
-                Group By Operator [GBY_112] (rows=1 width=344)
+                Group By Operator [GBY_78] (rows=1 width=344)
                   Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
-                <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
-                  PARTITION_ONLY_SHUFFLE [RS_111]
-                    Group By Operator [GBY_110] (rows=1 width=344)
+                <-Reducer 6 [CUSTOM_SIMPLE_EDGE]
+                  PARTITION_ONLY_SHUFFLE [RS_77]
+                    Group By Operator [GBY_76] (rows=1 width=344)
                       Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"]
-                      Group By Operator [GBY_109] (rows=1395035081047425024 width=1)
+                      Group By Operator [GBY_75] (rows=421645953 width=135)
                         Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
-                      <-Reducer 6 [SIMPLE_EDGE]
-                        SHUFFLE [RS_108]
+                      <-Reducer 5 [SIMPLE_EDGE]
+                        SHUFFLE [RS_74]
                           PartitionCols:_col0
-                          Group By Operator [GBY_107] (rows=1395035081047425024 width=1)
+                          Group By Operator [GBY_73] (rows=421645953 width=135)
                             Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4
-                            Select Operator [SEL_65] (rows=1395035081047425024 width=1)
+                            Select Operator [SEL_41] (rows=421645953 width=135)
                               Output:["_col4","_col5","_col6"]
-                              Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
+                              Filter Operator [FIL_40] (rows=421645953 width=135)
                                 predicate:_col16 is null
-                                Select Operator [SEL_63] (rows=2790070162094850048 width=1)
+                                Select Operator [SEL_39] (rows=843291907 width=135)
                                   Output:["_col4","_col5","_col6","_col16"]
-                                  Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1)
-                                    Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
-                                  <-Reducer 17 [SIMPLE_EDGE]
-                                    SHUFFLE [RS_61]
-                                      PartitionCols:_col0, _col1
-                                      Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
-                                        Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
-                                      <-Reducer 16 [SIMPLE_EDGE]
-                                        SHUFFLE [RS_45]
-                                          PartitionCols:_col0, _col1
-                                          Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
-                                            Output:["_col0","_col1"],keys:_col2, _col3
-                                            Select Operator [SEL_43] (rows=5072854730221289472 width=1)
-                                              Output:["_col2","_col3"]
-                                              Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
-                                                predicate:(_col2 <> _col0)
-                                                Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1)
-                                                  Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
-                                                <-Map 15 [SIMPLE_EDGE]
-                                                  PARTITION_ONLY_SHUFFLE [RS_39]
-                                                    PartitionCols:_col1
-                                                    Select Operator [SEL_20] (rows=287989836 width=135)
-                                                      Output:["_col0","_col1"]
-                                                      TableScan [TS_19] (rows=287989836 width=135)
-                                                        default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"]
-                                                <-Reducer 19 [SIMPLE_EDGE]
-                                                  SHUFFLE [RS_40]
-                                                    PartitionCols:_col1
-                                                    Select Operator [SEL_38] (rows=4611686018427387903 width=1)
-                                                      Output:["_col0","_col1"]
-                                                      Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
-                                                        Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
-                                                      <-Reducer 18 [SIMPLE_EDGE]
-                                                        SHUFFLE [RS_36]
-                                                          PartitionCols:_col0, _col1
-                                                          Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
-                                                            Output:["_col0","_col1"],keys:_col4, _col3
-                                                            Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1)
-                                                              Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
-                                                            <-Map 15 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_32]
-                                                                 Please refer to the previous Select Operator [SEL_20]
-                                                            <-Map 20 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_29]
-                                                                Select Operator [SEL_22] (rows=73049 width=4)
-                                                                  TableScan [TS_21] (rows=73049 width=1119)
-                                                                    default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
-                                                            <-Map 21 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_30]
-                                                                Select Operator [SEL_24] (rows=60 width=4)
-                                                                  TableScan [TS_23] (rows=60 width=2045)
-                                                                    default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE
-                                                            <-Map 22 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_31]
-                                                                Select Operator [SEL_26] (rows=40000000 width=4)
-                                                                  TableScan [TS_25] (rows=40000000 width=1014)
-                                                                    default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
-                                  <-Reducer 5 [SIMPLE_EDGE]
-                                    SHUFFLE [RS_60]
-                                      PartitionCols:_col3, _col4
-                                      Merge Join Operator [MERGEJOIN_118] (rows=421645953 width=135)
-                                        Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
-                                      <-Reducer 14 [SIMPLE_EDGE]
-                                        SHUFFLE [RS_58]
+                                  Merge Join Operator [MERGEJOIN_82] (rows=843291907 width=135)
+                                    Conds:RS_35._col4=RS_36._col0(Left Outer),RS_35._col4=RS_37._col1(Inner),Output:["_col3","_col4","_col5","_col6","_col14","_col15"],residual filter predicates:{(_col3 <> _col15)}
+                                  <-Reducer 13 [SIMPLE_EDGE]
+                                    SHUFFLE [RS_36]
+                                      PartitionCols:_col0
+                                      Select Operator [SEL_18] (rows=14399440 width=106)
+                                        Output:["_col0","_col1"]
+                                        Group By Operator [GBY_17] (rows=14399440 width=106)
+                                          Output:["_col0"],keys:KEY._col0
+                                        <-Map 12 [SIMPLE_EDGE]
+                                          SHUFFLE [RS_16]
+                                            PartitionCols:_col0
+                                            Group By Operator [GBY_15] (rows=28798881 width=106)
+                                              Output:["_col0"],keys:cr_order_number
+                                              Filter Operator [FIL_71] (rows=28798881 width=106)
+                                                predicate:cr_order_number is not null
+                                                TableScan [TS_12] (rows=28798881 width=106)
+                                                  default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"]
+                                  <-Reducer 15 [SIMPLE_EDGE]
+                                    SHUFFLE [RS_37]
+                                      PartitionCols:_col1
+                                      Select Operator [SEL_25] (rows=143994918 width=135)
+                                        Output:["_col0","_col1"]
+                                        Group By Operator [GBY_24] (rows=143994918 width=135)
+                                          Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+                                        <-Map 14 [SIMPLE_EDGE]
+                                          SHUFFLE [RS_23]
+                                            PartitionCols:_col0, _col1
+                                            Group By Operator [GBY_22] (rows=287989836 width=135)
+                                              Output:["_col0","_col1"],keys:cs_order_number, cs_warehouse_sk
+                                              Filter Operator [FIL_72] (rows=287989836 width=135)
+                                                predicate:(cs_order_number is not null and cs_warehouse_sk is not null)
+                                                TableScan [TS_19] (rows=287989836 width=135)
+                                                  default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"]
+                                  <-Reducer 4 [SIMPLE_EDGE]
+                                    SHUFFLE [RS_35]
+                                      PartitionCols:_col4
+                                      Merge Join Operator [MERGEJOIN_81] (rows=383314495 width=135)
+                                        Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
+                                      <-Map 11 [SIMPLE_EDGE]
+                                        SHUFFLE [RS_33]
                                           PartitionCols:_col0
-                                          Select Operator [SEL_18] (rows=14399440 width=106)
-                                            Output:["_col0","_col1"]
-                                            Group By Operator [GBY_17] (rows=14399440 width=106)
-                                              Output:["_col0"],keys:KEY._col0
-                                            <-Map 13 [SIMPLE_EDGE]
-                                              SHUFFLE [RS_16]
-                                                PartitionCols:_col0
-                                                Group By Operator [GBY_15] (rows=28798881 width=106)
-                                                  Output:["_col0"],keys:cr_order_number
-                                                  Filter Operator [FIL_104] (rows=28798881 width=106)
-                                                    predicate:cr_order_number is not null
-                                                    TableScan [TS_12] (rows=28798881 width=106)
-                                                      default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"]
-                                      <-Reducer 4 [SIMPLE_EDGE]
-                                        SHUFFLE [RS_57]
-                                          PartitionCols:_col4
-                                          Merge Join Operator [MERGEJOIN_116] (rows=383314495 width=135)
-                                            Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
-                                          <-Map 12 [SIMPLE_EDGE]
-                                            SHUFFLE [RS_55]
+                                          Select Operator [SEL_11] (rows=30 width=2045)
+                                            Output:["_col0"]
+                                            Filter Operator [FIL_70] (rows=30 width=2045)
+                                              predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null)
+                                              TableScan [TS_9] (rows=60 width=2045)
+                                                default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"]
+                                      <-Reducer 3 [SIMPLE_EDGE]
+                                        SHUFFLE [RS_32]
+                                          PartitionCols:_col2
+                                          Merge Join Operator [MERGEJOIN_80] (rows=348467716 width=135)
+                                            Conds:RS_29._col1=RS_30._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
+                                          <-Map 10 [SIMPLE_EDGE]
+                                            SHUFFLE [RS_30]
                                               PartitionCols:_col0
-                                              Select Operator [SEL_11] (rows=30 width=2045)
+                                              Select Operator [SEL_8] (rows=20000000 width=1014)
                                                 Output:["_col0"]
-                                                Filter Operator [FIL_103] (rows=30 width=2045)
-                                                  predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null)
-                                                  TableScan [TS_9] (rows=60 width=2045)
-                                                    default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"]
-                                          <-Reducer 3 [SIMPLE_EDGE]
-                                            SHUFFLE [RS_54]
-                                              PartitionCols:_col2
-                                              Merge Join Operator [MERGEJOIN_115] (rows=348467716 width=135)
-                                                Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
-                                              <-Map 11 [SIMPLE_EDGE]
-                                                SHUFFLE [RS_52]
+                                                Filter Operator [FIL_69] (rows=20000000 width=1014)
+                                                  predicate:((ca_state = 'NY') and ca_address_sk is not null)
+                                                  TableScan [TS_6] (rows=40000000 width=1014)
+                                                    default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+                                          <-Reducer 2 [SIMPLE_EDGE]
+                                            SHUFFLE [RS_29]
+                                              PartitionCols:_col1
+                                              Merge Join Operator [MERGEJOIN_79] (rows=316788826 width=135)
+                                                Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
+                                              <-Map 1 [SIMPLE_EDGE]
+                                                SHUFFLE [RS_26]
                                                   PartitionCols:_col0
-                                                  Select Operator [SEL_8] (rows=20000000 width=1014)
+                                                  Select Operator [SEL_2] (rows=287989836 width=135)
+                                                    Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+                                                    Filter Operator [FIL_67] (rows=287989836 width=135)
+                                                      predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null)
+                                                      TableScan [TS_0] (rows=287989836 width=135)
+                                                        default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"]
+                                              <-Map 9 [SIMPLE_EDGE]
+                                                SHUFFLE [RS_27]
+                                                  PartitionCols:_col0
+                                                  Select Operator [SEL_5] (rows=8116 width=1119)
                                                     Output:["_col0"]
-                                                    Filter Operator [FIL_102] (rows=20000000 width=1014)
-                                                      predicate:((ca_state = 'NY') and ca_address_sk is not null)
-                                                      TableScan [TS_6] (rows=40000000 width=1014)
-                                                        default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
-                                              <-Reducer 2 [SIMPLE_EDGE]
-                                                SHUFFLE [RS_51]
-                                                  PartitionCols:_col1
-                                                  Merge Join Operator [MERGEJOIN_114] (rows=316788826 width=135)
-                                                    Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
-                                                  <-Map 1 [SIMPLE_EDGE]
-                                                    SHUFFLE [RS_48]
-                                                      PartitionCols:_col0
-                                                      Select Operator [SEL_2] (rows=287989836 width=135)
-                                                        Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
-                                                        Filter Operator [FIL_100] (rows=287989836 width=135)
-                                                          predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null)
-                                                          TableScan [TS_0] (rows=287989836 width=135)
-                                                            default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"]
-                                                  <-Map 10 [SIMPLE_EDGE]
-                                                    SHUFFLE [RS_49]
-                                                      PartitionCols:_col0
-                                                      Select Operator [SEL_5] (rows=8116 width=1119)
-                                                        Output:["_col0"]
-                                                        Filter Operator [FIL_101] (rows=8116 width=1119)
-                                                          predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 2001-04-01 00:00:00.0 AND 2001-05-31 01:00:00.0 and d_date_sk is not null)
-                                                          TableScan [TS_3] (rows=73049 width=1119)
-                                                            default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
+                                                    Filter Operator [FIL_68] (rows=8116 width=1119)
+                                                      predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 2001-04-01 00:00:00.0 AND 2001-05-31 01:00:00.0 and d_date_sk is not null)
+                                                      TableScan [TS_3] (rows=73049 width=1119)
+                                                        default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
 

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/perf/query94.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query94.q.out b/ql/src/test/results/clientpositive/perf/query94.q.out
index 9cc934f..77fba1b 100644
--- a/ql/src/test/results/clientpositive/perf/query94.q.out
+++ b/ql/src/test/results/clientpositive/perf/query94.q.out
@@ -1,4 +1,3 @@
-Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product
 PREHOOK: query: explain
 select  
    count(distinct ws_order_number) as `order count`
@@ -58,180 +57,132 @@ POSTHOOK: type: QUERY
 Plan optimized by CBO.
 
 Vertex dependency in root stage
-Reducer 14 <- Map 13 (SIMPLE_EDGE)
-Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
-Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE)
-Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
-Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 17 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
-Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+Reducer 13 <- Map 12 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
+Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
 
 Stage-0
   Fetch Operator
     limit:-1
     Stage-1
-      Reducer 9
-      File Output Operator [FS_74]
-        Limit [LIM_72] (rows=1 width=344)
+      Reducer 8
+      File Output Operator [FS_50]
+        Limit [LIM_48] (rows=1 width=344)
           Number of rows:100
-          Select Operator [SEL_71] (rows=1 width=344)
+          Select Operator [SEL_47] (rows=1 width=344)
             Output:["_col0","_col1","_col2"]
-          <-Reducer 8 [SIMPLE_EDGE]
-            SHUFFLE [RS_70]
-              Select Operator [SEL_69] (rows=1 width=344)
+          <-Reducer 7 [SIMPLE_EDGE]
+            SHUFFLE [RS_46]
+              Select Operator [SEL_45] (rows=1 width=344)
                 Output:["_col1","_col2","_col3"]
-                Group By Operator [GBY_112] (rows=1 width=344)
+                Group By Operator [GBY_78] (rows=1 width=344)
                   Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
-                <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
-                  PARTITION_ONLY_SHUFFLE [RS_111]
-                    Group By Operator [GBY_110] (rows=1 width=344)
+                <-Reducer 6 [CUSTOM_SIMPLE_EDGE]
+                  PARTITION_ONLY_SHUFFLE [RS_77]
+                    Group By Operator [GBY_76] (rows=1 width=344)
                       Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"]
-                      Group By Operator [GBY_109] (rows=1395035081047425024 width=1)
+                      Group By Operator [GBY_75] (rows=210834322 width=135)
                         Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
-                      <-Reducer 6 [SIMPLE_EDGE]
-                        SHUFFLE [RS_108]
+                      <-Reducer 5 [SIMPLE_EDGE]
+                        SHUFFLE [RS_74]
                           PartitionCols:_col0
-                          Group By Operator [GBY_107] (rows=1395035081047425024 width=1)
+                          Group By Operator [GBY_73] (rows=210834322 width=135)
                             Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4
-                            Select Operator [SEL_65] (rows=1395035081047425024 width=1)
+                            Select Operator [SEL_41] (rows=210834322 width=135)
                               Output:["_col4","_col5","_col6"]
-                              Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
+                              Filter Operator [FIL_40] (rows=210834322 width=135)
                                 predicate:_col16 is null
-                                Select Operator [SEL_63] (rows=2790070162094850048 width=1)
+                                Select Operator [SEL_39] (rows=421668645 width=135)
                                   Output:["_col4","_col5","_col6","_col16"]
-                                  Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1)
-                                    Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
-                                  <-Reducer 17 [SIMPLE_EDGE]
-                                    SHUFFLE [RS_61]
-                                      PartitionCols:_col0, _col1
-                                      Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
-                                        Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
-                                      <-Reducer 16 [SIMPLE_EDGE]
-                                        SHUFFLE [RS_45]
-                                          PartitionCols:_col0, _col1
-                                          Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
-                                            Output:["_col0","_col1"],keys:_col2, _col3
-                                            Select Operator [SEL_43] (rows=5072854730221289472 width=1)
-                                              Output:["_col2","_col3"]
-                                              Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
-                                                predicate:(_col2 <> _col0)
-                                                Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1)
-                                                  Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
-                                                <-Map 15 [SIMPLE_EDGE]
-                                                  PARTITION_ONLY_SHUFFLE [RS_39]
-                                                    PartitionCols:_col1
-                                                    Select Operator [SEL_20] (rows=144002668 width=135)
-                                                      Output:["_col0","_col1"]
-                                                      TableScan [TS_19] (rows=144002668 width=135)
-                                                        default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
-                                                <-Reducer 19 [SIMPLE_EDGE]
-                                                  SHUFFLE [RS_40]
-                                                    PartitionCols:_col1
-                                                    Select Operator [SEL_38] (rows=4611686018427387903 width=1)
-                                                      Output:["_col0","_col1"]
-                                                      Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
-                                                        Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
-                                                      <-Reducer 18 [SIMPLE_EDGE]
-                                                        SHUFFLE [RS_36]
-                                                          PartitionCols:_col0, _col1
-                                                          Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
-                                                            Output:["_col0","_col1"],keys:_col4, _col3
-                                                            Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1)
-                                                              Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
-                                                            <-Map 15 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_32]
-                                                                 Please refer to the previous Select Operator [SEL_20]
-                                                            <-Map 20 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_29]
-                                                                Select Operator [SEL_22] (rows=73049 width=4)
-                                                                  TableScan [TS_21] (rows=73049 width=1119)
-                                                                    default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
-                                                            <-Map 21 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_30]
-                                                                Select Operator [SEL_24] (rows=84 width=4)
-                                                                  TableScan [TS_23] (rows=84 width=1850)
-                                                                    default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE
-                                                            <-Map 22 [CUSTOM_SIMPLE_EDGE]
-                                                              PARTITION_ONLY_SHUFFLE [RS_31]
-                                                                Select Operator [SEL_26] (rows=40000000 width=4)
-                                                                  TableScan [TS_25] (rows=40000000 width=1014)
-                                                                    default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
-                                  <-Reducer 5 [SIMPLE_EDGE]
-                                    SHUFFLE [RS_60]
-                                      PartitionCols:_col3, _col4
-                                      Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135)
-                                        Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
-                                      <-Reducer 14 [SIMPLE_EDGE]
-                                        SHUFFLE [RS_58]
+                                  Merge Join Operator [MERGEJOIN_82] (rows=421668645 width=135)
+                                    Conds:RS_35._col4=RS_36._col0(Left Outer),RS_35._col4=RS_37._col1(Inner),Output:["_col3","_col4","_col5","_col6","_col14","_col15"],residual filter predicates:{(_col3 <> _col15)}
+                                  <-Reducer 13 [SIMPLE_EDGE]
+                                    SHUFFLE [RS_36]
+                                      PartitionCols:_col0
+                                      Select Operator [SEL_18] (rows=7199233 width=92)
+                                        Output:["_col0","_col1"]
+                                        Group By Operator [GBY_17] (rows=7199233 width=92)
+                                          Output:["_col0"],keys:KEY._col0
+                                        <-Map 12 [SIMPLE_EDGE]
+                                          SHUFFLE [RS_16]
+                                            PartitionCols:_col0
+                                            Group By Operator [GBY_15] (rows=14398467 width=92)
+                                              Output:["_col0"],keys:wr_order_number
+                                              Filter Operator [FIL_71] (rows=14398467 width=92)
+                                                predicate:wr_order_number is not null
+                                                TableScan [TS_12] (rows=14398467 width=92)
+                                                  default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
+                                  <-Reducer 15 [SIMPLE_EDGE]
+                                    SHUFFLE [RS_37]
+                                      PartitionCols:_col1
+                                      Select Operator [SEL_25] (rows=72001334 width=135)
+                                        Output:["_col0","_col1"]
+                                        Group By Operator [GBY_24] (rows=72001334 width=135)
+                                          Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+                                        <-Map 14 [SIMPLE_EDGE]
+                                          SHUFFLE [RS_23]
+                                            PartitionCols:_col0, _col1
+                                            Group By Operator [GBY_22] (rows=144002668 width=135)
+                                              Output:["_col0","_col1"],keys:ws_order_number, ws_warehouse_sk
+                                              Filter Operator [FIL_72] (rows=144002668 width=135)
+                                                predicate:(ws_order_number is not null and ws_warehouse_sk is not null)
+                                                TableScan [TS_19] (rows=144002668 width=135)
+                                                  default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
+                                  <-Reducer 4 [SIMPLE_EDGE]
+                                    SHUFFLE [RS_35]
+                                      PartitionCols:_col4
+                                      Merge Join Operator [MERGEJOIN_81] (rows=191667562 width=135)
+                                        Conds:RS_32._col2=RS_33._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
+                                      <-Map 11 [SIMPLE_EDGE]
+                                        SHUFFLE [RS_33]
                                           PartitionCols:_col0
-                                          Select Operator [SEL_18] (rows=7199233 width=92)
-                                            Output:["_col0","_col1"]
-                                            Group By Operator [GBY_17] (rows=7199233 width=92)
-                                              Output:["_col0"],keys:KEY._col0
-                                            <-Map 13 [SIMPLE_EDGE]
-                                              SHUFFLE [RS_16]
-                                                PartitionCols:_col0
-                                                Group By Operator [GBY_15] (rows=14398467 width=92)
-                                                  Output:["_col0"],keys:wr_order_number
-                                                  Filter Operator [FIL_104] (rows=14398467 width=92)
-                                                    predicate:wr_order_number is not null
-                                                    TableScan [TS_12] (rows=14398467 width=92)
-                                                      default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
-                                      <-Reducer 4 [SIMPLE_EDGE]
-                                        SHUFFLE [RS_57]
-                                          PartitionCols:_col4
-                                          Merge Join Operator [MERGEJOIN_116] (rows=191667562 width=135)
-                                            Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
-                                          <-Map 12 [SIMPLE_EDGE]
-                                            SHUFFLE [RS_55]
+                                          Select Operator [SEL_11] (rows=42 width=1850)
+                                            Output:["_col0"]
+                                            Filter Operator [FIL_70] (rows=42 width=1850)
+                                              predicate:((web_company_name = 'pri') and web_site_sk is not null)
+                                              TableScan [TS_9] (rows=84 width=1850)
+                                                default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
+                                      <-Reducer 3 [SIMPLE_EDGE]
+                                        SHUFFLE [RS_32]
+                                          PartitionCols:_col2
+                                          Merge Join Operator [MERGEJOIN_80] (rows=174243235 width=135)
+                                            Conds:RS_29._col1=RS_30._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
+                                          <-Map 10 [SIMPLE_EDGE]
+                                            SHUFFLE [RS_30]
                                               PartitionCols:_col0
-                                              Select Operator [SEL_11] (rows=42 width=1850)
+                                              Select Operator [SEL_8] (rows=20000000 width=1014)
                                                 Output:["_col0"]
-                                                Filter Operator [FIL_103] (rows=42 width=1850)
-                                                  predicate:((web_company_name = 'pri') and web_site_sk is not null)
-                                                  TableScan [TS_9] (rows=84 width=1850)
-                                                    default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
-                                          <-Reducer 3 [SIMPLE_EDGE]
-                                            SHUFFLE [RS_54]
-                                              PartitionCols:_col2
-                                              Merge Join Operator [MERGEJOIN_115] (rows=174243235 width=135)
-                                                Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
-                                              <-Map 11 [SIMPLE_EDGE]
-                                                SHUFFLE [RS_52]
+                                                Filter Operator [FIL_69] (rows=20000000 width=1014)
+                                                  predicate:((ca_state = 'TX') and ca_address_sk is not null)
+                                                  TableScan [TS_6] (rows=40000000 width=1014)
+                                                    default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+                                          <-Reducer 2 [SIMPLE_EDGE]
+                                            SHUFFLE [RS_29]
+                                              PartitionCols:_col1
+                                              Merge Join Operator [MERGEJOIN_79] (rows=158402938 width=135)
+                                                Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
+                                              <-Map 1 [SIMPLE_EDGE]
+                                                SHUFFLE [RS_26]
                                                   PartitionCols:_col0
-                                                  Select Operator [SEL_8] (rows=20000000 width=1014)
+                                                  Select Operator [SEL_2] (rows=144002668 width=135)
+                                                    Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+                                                    Filter Operator [FIL_67] (rows=144002668 width=135)
+                                                      predicate:(ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null)
+                                                      TableScan [TS_0] (rows=144002668 width=135)
+                                                        default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
+                                              <-Map 9 [SIMPLE_EDGE]
+                                                SHUFFLE [RS_27]
+                                                  PartitionCols:_col0
+                                                  Select Operator [SEL_5] (rows=8116 width=1119)
                                                     Output:["_col0"]
-                                                    Filter Operator [FIL_102] (rows=20000000 width=1014)
-                                                      predicate:((ca_state = 'TX') and ca_address_sk is not null)
-                                                      TableScan [TS_6] (rows=40000000 width=1014)
-                                                        default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
-                                              <-Reducer 2 [SIMPLE_EDGE]
-                                                SHUFFLE [RS_51]
-                                                  PartitionCols:_col1
-                                                  Merge Join Operator [MERGEJOIN_114] (rows=158402938 width=135)
-                                                    Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
-                                                  <-Map 1 [SIMPLE_EDGE]
-                                                    SHUFFLE [RS_48]
-                                                      PartitionCols:_col0
-                                                      Select Operator [SEL_2] (rows=144002668 width=135)
-                                                        Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
-                                                        Filter Operator [FIL_100] (rows=144002668 width=135)
-                                                          predicate:(ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null)
-                                                          TableScan [TS_0] (rows=144002668 width=135)
-                                                            default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
-                                                  <-Map 10 [SIMPLE_EDGE]
-                                                    SHUFFLE [RS_49]
-                                                      PartitionCols:_col0
-                                                      Select Operator [SEL_5] (rows=8116 width=1119)
-                                                        Output:["_col0"]
-                                                        Filter Operator [FIL_101] (rows=8116 width=1119)
-                                                          predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
-                                                          TableScan [TS_3] (rows=73049 width=1119)
-                                                            default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
+                                                    Filter Operator [FIL_68] (rows=8116 width=1119)
+                                                      predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
+                                                      TableScan [TS_3] (rows=73049 width=1119)
+                                                        default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
 

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
index 3467215..5b57036 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
@@ -122,22 +122,22 @@ STAGE PLANS:
                   alias: lineitem
                   Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-                    Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+                    Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: l_orderkey (type: int), l_linenumber (type: int)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         keys: _col0 (type: int), _col1 (type: int)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: int), _col1 (type: int)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
-                          Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
         Reducer 2 
             Reduce Operator Tree:
               Join Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
index 8768b45..2c86196 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
@@ -47,22 +47,22 @@ STAGE PLANS:
                   alias: a
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((value = value) and (key = key) and (value > 'val_9')) (type: boolean)
-                    Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((value > 'val_9') and key is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
         Reducer 2 
             Reduce Operator Tree:
               Join Operator


[2/6] hive git commit: HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_in_having.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_in_having.q.out b/ql/src/test/results/clientpositive/subquery_in_having.q.out
index 2bb812c..19142e3 100644
--- a/ql/src/test/results/clientpositive/subquery_in_having.q.out
+++ b/ql/src/test/results/clientpositive/subquery_in_having.q.out
@@ -1776,11 +1776,14 @@ POSTHOOK: type: QUERY
 POSTHOOK: Output: default@src_null
 POSTHOOK: Lineage: src_null.key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
 POSTHOOK: Lineage: src_null.value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
-Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[186][bigTable=?] in task 'Stage-8:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[187][bigTable=?] in task 'Stage-9:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[188][bigTable=?] in task 'Stage-14:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[189][bigTable=?] in task 'Stage-15:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[146][bigTable=?] in task 'Stage-18:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[147][bigTable=?] in task 'Stage-7:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[148][bigTable=?] in task 'Stage-21:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[32][tables = [$hdt$_2, $hdt$_3]] in Stage 'Stage-8:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[149][bigTable=?] in task 'Stage-12:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[150][bigTable=?] in task 'Stage-24:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[72][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-13:MAPRED' is a cross product
 PREHOOK: query: explain
 select key, value, count(*)
 from src_null b
@@ -1796,39 +1799,44 @@ group by key, value
 having count(*) not in (select count(*) from src_null s1 where s1.key > '9' and s1.value <> b.value group by s1.key )
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-6 is a root stage
-  Stage-27 depends on stages: Stage-6
-  Stage-1 depends on stages: Stage-27
-  Stage-2 depends on stages: Stage-1
-  Stage-20 depends on stages: Stage-2, Stage-8 , consists of Stage-26, Stage-3
+  Stage-5 is a root stage
+  Stage-19 depends on stages: Stage-5 , consists of Stage-27, Stage-1
+  Stage-27 has a backup stage: Stage-1
+  Stage-18 depends on stages: Stage-27
+  Stage-2 depends on stages: Stage-1, Stage-18
+  Stage-17 depends on stages: Stage-2, Stage-7 , consists of Stage-26, Stage-3
   Stage-26 has a backup stage: Stage-3
-  Stage-19 depends on stages: Stage-26
-  Stage-4 depends on stages: Stage-3, Stage-14, Stage-19
+  Stage-16 depends on stages: Stage-26
+  Stage-4 depends on stages: Stage-3, Stage-12, Stage-16
   Stage-3
-  Stage-12 is a root stage
-  Stage-29 depends on stages: Stage-12
-  Stage-9 depends on stages: Stage-29
-  Stage-10 depends on stages: Stage-9
-  Stage-28 depends on stages: Stage-10
-  Stage-8 depends on stages: Stage-28
-  Stage-18 is a root stage
-  Stage-31 depends on stages: Stage-18
-  Stage-15 depends on stages: Stage-31
-  Stage-16 depends on stages: Stage-15
-  Stage-30 depends on stages: Stage-16
-  Stage-14 depends on stages: Stage-30
+  Stage-1
+  Stage-10 is a root stage
+  Stage-22 depends on stages: Stage-10 , consists of Stage-29, Stage-8
+  Stage-29 has a backup stage: Stage-8
+  Stage-21 depends on stages: Stage-29
+  Stage-9 depends on stages: Stage-8, Stage-21
+  Stage-28 depends on stages: Stage-9
+  Stage-7 depends on stages: Stage-28
+  Stage-8
+  Stage-15 is a root stage
+  Stage-25 depends on stages: Stage-15 , consists of Stage-31, Stage-13
+  Stage-31 has a backup stage: Stage-13
+  Stage-24 depends on stages: Stage-31
+  Stage-14 depends on stages: Stage-13, Stage-24
+  Stage-30 depends on stages: Stage-14
+  Stage-12 depends on stages: Stage-30
+  Stage-13
   Stage-0 depends on stages: Stage-4
 
 STAGE PLANS:
-  Stage: Stage-6
+  Stage: Stage-5
     Map Reduce
       Map Operator Tree:
           TableScan
-            alias: b
+            alias: src_null
             Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: value (type: string)
-              outputColumnNames: value
+            Filter Operator
+              predicate: value is not null (type: boolean)
               Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: value (type: string)
@@ -1846,34 +1854,35 @@ STAGE PLANS:
           mode: mergepartial
           outputColumnNames: _col0
           Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            table:
-                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          Select Operator
+            expressions: _col0 (type: string), true (type: boolean)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-19
+    Conditional Operator
 
   Stage: Stage-27
     Map Reduce Local Work
       Alias -> Map Local Tables:
-        $hdt$_0:$hdt$_1:$hdt$_1:src_null 
+        $hdt$_0:$INTNAME 
           Fetch Operator
             limit: -1
       Alias -> Map Local Operator Tree:
-        $hdt$_0:$hdt$_1:$hdt$_1:src_null 
+        $hdt$_0:$INTNAME 
           TableScan
-            alias: src_null
-            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: value (type: string)
-              outputColumnNames: _col0
-              Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-              HashTable Sink Operator
-                keys:
-                  0 
-                  1 
+            HashTable Sink Operator
+              keys:
+                0 
+                1 
 
-  Stage: Stage-1
+  Stage: Stage-18
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -1883,110 +1892,36 @@ STAGE PLANS:
               expressions: key (type: string), value (type: string)
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-              Reduce Output Operator
-                key expressions: _col1 (type: string)
-                sort order: +
-                Map-reduce partition columns: _col1 (type: string)
-                Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-                value expressions: _col0 (type: string)
-          TableScan
-            Map Join Operator
-              condition map:
-                   Inner Join 0 to 1
-              keys:
-                0 
-                1 
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-              Filter Operator
-                predicate: (_col0 <> _col1) (type: boolean)
-                Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: string)
-                  outputColumnNames: _col1
-                  Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                  Group By Operator
-                    keys: _col1 (type: string)
-                    mode: hash
-                    outputColumnNames: _col0
-                    Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
-                      Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
+              Map Join Operator
+                condition map:
+                     Left Outer Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2, _col3
+                residual filter predicates: {(_col1 <> _col2)}
+                Statistics: Num rows: 841 Data size: 254011 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: _col3 is null (type: boolean)
+                  Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count()
+                      keys: _col1 (type: string), _col0 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1, _col2
+                      Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                      File Output Operator
+                        compressed: false
+                        table:
+                            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
       Local Work:
         Map Reduce Local Work
-      Reduce Operator Tree:
-        Demux Operator
-          Statistics: Num rows: 1711 Data size: 345062 Basic stats: COMPLETE Column stats: NONE
-          Mux Operator
-            Statistics: Num rows: 2566 Data size: 517492 Basic stats: COMPLETE Column stats: NONE
-            Join Operator
-              condition map:
-                   Left Outer Join 0 to 1
-              keys:
-                0 _col1 (type: string)
-                1 _col0 (type: string)
-              outputColumnNames: _col0, _col1, _col3
-              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-              Filter Operator
-                predicate: _col3 is null (type: boolean)
-                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                Select Operator
-                  expressions: _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Group By Operator
-                    aggregations: count()
-                    keys: _col1 (type: string), _col0 (type: string)
-                    mode: hash
-                    outputColumnNames: _col0, _col1, _col2
-                    Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      table:
-                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-          Group By Operator
-            keys: KEY._col0 (type: string)
-            mode: mergepartial
-            outputColumnNames: _col0
-            Statistics: Num rows: 855 Data size: 172430 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: _col0 (type: string), true (type: boolean)
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 855 Data size: 172430 Basic stats: COMPLETE Column stats: NONE
-              Mux Operator
-                Statistics: Num rows: 2566 Data size: 517492 Basic stats: COMPLETE Column stats: NONE
-                Join Operator
-                  condition map:
-                       Left Outer Join 0 to 1
-                  keys:
-                    0 _col1 (type: string)
-                    1 _col0 (type: string)
-                  outputColumnNames: _col0, _col1, _col3
-                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Filter Operator
-                    predicate: _col3 is null (type: boolean)
-                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Group By Operator
-                        aggregations: count()
-                        keys: _col1 (type: string), _col0 (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2
-                        Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-2
     Map Reduce
@@ -1996,7 +1931,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col2 (type: bigint)
       Reduce Operator Tree:
         Group By Operator
@@ -2004,11 +1939,11 @@ STAGE PLANS:
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2
-          Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+          Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col1 (type: string), _col0 (type: string), _col2 (type: bigint)
             outputColumnNames: _col0, _col1, _col2
-            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+            Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
               table:
@@ -2016,7 +1951,7 @@ STAGE PLANS:
                   output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                   serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-20
+  Stage: Stage-17
     Conditional Operator
 
   Stage: Stage-26
@@ -2033,7 +1968,7 @@ STAGE PLANS:
                 0 _col1 (type: string)
                 1 _col0 (type: string)
 
-  Stage: Stage-19
+  Stage: Stage-16
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2044,7 +1979,7 @@ STAGE PLANS:
                 0 _col1 (type: string)
                 1 _col0 (type: string)
               outputColumnNames: _col0, _col1, _col2, _col4, _col5
-              Statistics: Num rows: 2 Data size: 443 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 259 Data size: 130759 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
                 table:
@@ -2062,19 +1997,19 @@ STAGE PLANS:
               key expressions: _col1 (type: string), _col2 (type: bigint)
               sort order: ++
               Map-reduce partition columns: _col1 (type: string), _col2 (type: bigint)
-              Statistics: Num rows: 2 Data size: 443 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 259 Data size: 130759 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col0 (type: string), _col4 (type: bigint), _col5 (type: bigint)
           TableScan
             Reduce Output Operator
               key expressions: _col0 (type: string), _col1 (type: bigint)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint)
-              Statistics: Num rows: 4 Data size: 806 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 472 Data size: 237744 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Demux Operator
-          Statistics: Num rows: 6 Data size: 1249 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 731 Data size: 368503 Basic stats: COMPLETE Column stats: NONE
           Mux Operator
-            Statistics: Num rows: 9 Data size: 1873 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 1096 Data size: 552502 Basic stats: COMPLETE Column stats: NONE
             Join Operator
               condition map:
                    Left Outer Join 0 to 1
@@ -2101,20 +2036,20 @@ STAGE PLANS:
             keys: KEY._col0 (type: string), KEY._col1 (type: bigint)
             mode: mergepartial
             outputColumnNames: _col0, _col1
-            Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 365 Data size: 183999 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               expressions: _col1 (type: bigint), _col0 (type: string)
               outputColumnNames: _col0, _col1
-              Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 365 Data size: 183999 Basic stats: COMPLETE Column stats: NONE
               Filter Operator
                 predicate: _col0 is not null (type: boolean)
-                Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 365 Data size: 183999 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col0 (type: bigint), _col1 (type: string), true (type: boolean)
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 3 Data size: 624 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 365 Data size: 183999 Basic stats: COMPLETE Column stats: NONE
                   Mux Operator
-                    Statistics: Num rows: 9 Data size: 1873 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 1096 Data size: 552502 Basic stats: COMPLETE Column stats: NONE
                     Join Operator
                       condition map:
                            Left Outer Join 0 to 1
@@ -2146,14 +2081,14 @@ STAGE PLANS:
               key expressions: _col1 (type: string)
               sort order: +
               Map-reduce partition columns: _col1 (type: string)
-              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col0 (type: string), _col2 (type: bigint)
           TableScan
             Reduce Output Operator
               key expressions: _col0 (type: string)
               sort order: +
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 2 Data size: 403 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 236 Data size: 118872 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: bigint), _col2 (type: bigint)
       Reduce Operator Tree:
         Join Operator
@@ -2163,7 +2098,7 @@ STAGE PLANS:
             0 _col1 (type: string)
             1 _col0 (type: string)
           outputColumnNames: _col0, _col1, _col2, _col4, _col5
-          Statistics: Num rows: 2 Data size: 443 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 259 Data size: 130759 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
             table:
@@ -2171,15 +2106,63 @@ STAGE PLANS:
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                 serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-12
+  Stage: Stage-1
     Map Reduce
       Map Operator Tree:
           TableScan
             alias: b
-            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: key (type: string), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                sort order: 
+                Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+                value expressions: _col0 (type: string), _col1 (type: string)
+          TableScan
+            Reduce Output Operator
+              sort order: 
+              Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
+              value expressions: _col0 (type: string), _col1 (type: boolean)
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Outer Join 0 to 1
+          keys:
+            0 
+            1 
+          outputColumnNames: _col0, _col1, _col2, _col3
+          residual filter predicates: {(_col1 <> _col2)}
+          Statistics: Num rows: 841 Data size: 254011 Basic stats: COMPLETE Column stats: NONE
+          Filter Operator
+            predicate: _col3 is null (type: boolean)
+            Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
             Select Operator
-              expressions: value (type: string)
-              outputColumnNames: value
+              expressions: _col0 (type: string), _col1 (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+              Group By Operator
+                aggregations: count()
+                keys: _col1 (type: string), _col0 (type: string)
+                mode: hash
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-10
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src_null
+            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: value is not null (type: boolean)
               Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: value (type: string)
@@ -2197,34 +2180,35 @@ STAGE PLANS:
           mode: mergepartial
           outputColumnNames: _col0
           Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            table:
-                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          Select Operator
+            expressions: _col0 (type: string), true (type: boolean)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-22
+    Conditional Operator
 
   Stage: Stage-29
     Map Reduce Local Work
       Alias -> Map Local Tables:
-        $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:src_null 
+        $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$INTNAME 
           Fetch Operator
             limit: -1
       Alias -> Map Local Operator Tree:
-        $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:src_null 
+        $hdt$_1:$hdt$_1:$hdt$_2:$hdt$_2:$INTNAME 
           TableScan
-            alias: src_null
-            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: value (type: string)
-              outputColumnNames: _col0
-              Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-              HashTable Sink Operator
-                keys:
-                  0 
-                  1 
+            HashTable Sink Operator
+              keys:
+                0 
+                1 
 
-  Stage: Stage-9
+  Stage: Stage-21
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2234,110 +2218,37 @@ STAGE PLANS:
               expressions: key (type: string), value (type: string)
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-              Reduce Output Operator
-                key expressions: _col1 (type: string)
-                sort order: +
-                Map-reduce partition columns: _col1 (type: string)
-                Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-                value expressions: _col0 (type: string)
-          TableScan
-            Map Join Operator
-              condition map:
-                   Inner Join 0 to 1
-              keys:
-                0 
-                1 
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-              Filter Operator
-                predicate: (_col0 <> _col1) (type: boolean)
-                Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: string)
-                  outputColumnNames: _col1
-                  Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                  Group By Operator
-                    keys: _col1 (type: string)
-                    mode: hash
-                    outputColumnNames: _col0
-                    Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
-                      Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-      Local Work:
-        Map Reduce Local Work
-      Reduce Operator Tree:
-        Demux Operator
-          Statistics: Num rows: 1711 Data size: 345062 Basic stats: COMPLETE Column stats: NONE
-          Mux Operator
-            Statistics: Num rows: 2566 Data size: 517492 Basic stats: COMPLETE Column stats: NONE
-            Join Operator
-              condition map:
-                   Left Outer Join 0 to 1
-              keys:
-                0 _col1 (type: string)
-                1 _col0 (type: string)
-              outputColumnNames: _col0, _col1, _col3
-              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-              Filter Operator
-                predicate: _col3 is null (type: boolean)
-                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                Select Operator
-                  expressions: _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Group By Operator
-                    keys: _col1 (type: string), _col0 (type: string)
-                    mode: hash
+              Map Join Operator
+                condition map:
+                     Left Outer Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2, _col3
+                residual filter predicates: {(_col1 <> _col2)}
+                Statistics: Num rows: 841 Data size: 254011 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: _col3 is null (type: boolean)
+                  Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string)
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      table:
-                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-          Group By Operator
-            keys: KEY._col0 (type: string)
-            mode: mergepartial
-            outputColumnNames: _col0
-            Statistics: Num rows: 855 Data size: 172430 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: _col0 (type: string), true (type: boolean)
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 855 Data size: 172430 Basic stats: COMPLETE Column stats: NONE
-              Mux Operator
-                Statistics: Num rows: 2566 Data size: 517492 Basic stats: COMPLETE Column stats: NONE
-                Join Operator
-                  condition map:
-                       Left Outer Join 0 to 1
-                  keys:
-                    0 _col1 (type: string)
-                    1 _col0 (type: string)
-                  outputColumnNames: _col0, _col1, _col3
-                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Filter Operator
-                    predicate: _col3 is null (type: boolean)
-                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
+                    Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      keys: _col1 (type: string), _col0 (type: string)
+                      mode: hash
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Group By Operator
-                        keys: _col1 (type: string), _col0 (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                      Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                      File Output Operator
+                        compressed: false
+                        table:
+                            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Local Work:
+        Map Reduce Local Work
 
-  Stage: Stage-10
+  Stage: Stage-9
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2345,22 +2256,22 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+          Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string)
             outputColumnNames: _col1
-            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+            Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               keys: _col1 (type: string)
               mode: complete
               outputColumnNames: _col0
-              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Statistics: Num rows: 105 Data size: 31713 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
                 table:
@@ -2391,7 +2302,7 @@ STAGE PLANS:
                     0 
                     1 
 
-  Stage: Stage-8
+  Stage: Stage-7
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2402,25 +2313,25 @@ STAGE PLANS:
                 0 
                 1 
               outputColumnNames: _col0, _col1, _col2
-              Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
               Filter Operator
                 predicate: (_col1 <> _col2) (type: boolean)
-                Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col0 (type: string), _col2 (type: string)
                   outputColumnNames: _col0, _col2
-                  Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                   Group By Operator
                     aggregations: count()
                     keys: _col2 (type: string), _col0 (type: string)
                     mode: hash
                     outputColumnNames: _col0, _col1, _col2
-                    Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
                       key expressions: _col0 (type: string), _col1 (type: string)
                       sort order: ++
                       Map-reduce partition columns: _col0 (type: string)
-                      Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col2 (type: bigint)
       Local Work:
         Map Reduce Local Work
@@ -2430,17 +2341,17 @@ STAGE PLANS:
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2
-          Statistics: Num rows: 4 Data size: 806 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 472 Data size: 237744 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string), _col2 (type: bigint)
             outputColumnNames: _col1, _col2
-            Statistics: Num rows: 4 Data size: 806 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 472 Data size: 237744 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               aggregations: count(), count(_col2)
               keys: _col1 (type: string)
               mode: complete
               outputColumnNames: _col0, _col1, _col2
-              Statistics: Num rows: 2 Data size: 403 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 236 Data size: 118872 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
                 table:
@@ -2448,15 +2359,62 @@ STAGE PLANS:
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                     serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-18
+  Stage: Stage-8
     Map Reduce
       Map Operator Tree:
           TableScan
             alias: b
-            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
             Select Operator
-              expressions: value (type: string)
-              outputColumnNames: value
+              expressions: key (type: string), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                sort order: 
+                Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+                value expressions: _col0 (type: string), _col1 (type: string)
+          TableScan
+            Reduce Output Operator
+              sort order: 
+              Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
+              value expressions: _col0 (type: string), _col1 (type: boolean)
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Outer Join 0 to 1
+          keys:
+            0 
+            1 
+          outputColumnNames: _col0, _col1, _col2, _col3
+          residual filter predicates: {(_col1 <> _col2)}
+          Statistics: Num rows: 841 Data size: 254011 Basic stats: COMPLETE Column stats: NONE
+          Filter Operator
+            predicate: _col3 is null (type: boolean)
+            Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: _col0 (type: string), _col1 (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+              Group By Operator
+                keys: _col1 (type: string), _col0 (type: string)
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-15
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src_null
+            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: value is not null (type: boolean)
               Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: value (type: string)
@@ -2474,34 +2432,35 @@ STAGE PLANS:
           mode: mergepartial
           outputColumnNames: _col0
           Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
-          File Output Operator
-            compressed: false
-            table:
-                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          Select Operator
+            expressions: _col0 (type: string), true (type: boolean)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-25
+    Conditional Operator
 
   Stage: Stage-31
     Map Reduce Local Work
       Alias -> Map Local Tables:
-        $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$hdt$_4:$hdt$_4:src_null 
+        $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$INTNAME 
           Fetch Operator
             limit: -1
       Alias -> Map Local Operator Tree:
-        $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$hdt$_4:$hdt$_4:src_null 
+        $hdt$_2:$hdt$_2:$hdt$_2:$hdt$_2:$hdt$_3:$hdt$_3:$INTNAME 
           TableScan
-            alias: src_null
-            Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: value (type: string)
-              outputColumnNames: _col0
-              Statistics: Num rows: 58 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-              HashTable Sink Operator
-                keys:
-                  0 
-                  1 
+            HashTable Sink Operator
+              keys:
+                0 
+                1 
 
-  Stage: Stage-15
+  Stage: Stage-24
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2511,110 +2470,37 @@ STAGE PLANS:
               expressions: key (type: string), value (type: string)
               outputColumnNames: _col0, _col1
               Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-              Reduce Output Operator
-                key expressions: _col1 (type: string)
-                sort order: +
-                Map-reduce partition columns: _col1 (type: string)
-                Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
-                value expressions: _col0 (type: string)
-          TableScan
-            Map Join Operator
-              condition map:
-                   Inner Join 0 to 1
-              keys:
-                0 
-                1 
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-              Filter Operator
-                predicate: (_col0 <> _col1) (type: boolean)
-                Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: string)
-                  outputColumnNames: _col1
-                  Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                  Group By Operator
-                    keys: _col1 (type: string)
-                    mode: hash
-                    outputColumnNames: _col0
-                    Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: string)
-                      Statistics: Num rows: 1682 Data size: 339242 Basic stats: COMPLETE Column stats: NONE
-      Local Work:
-        Map Reduce Local Work
-      Reduce Operator Tree:
-        Demux Operator
-          Statistics: Num rows: 1711 Data size: 345062 Basic stats: COMPLETE Column stats: NONE
-          Mux Operator
-            Statistics: Num rows: 2566 Data size: 517492 Basic stats: COMPLETE Column stats: NONE
-            Join Operator
-              condition map:
-                   Left Outer Join 0 to 1
-              keys:
-                0 _col1 (type: string)
-                1 _col0 (type: string)
-              outputColumnNames: _col0, _col1, _col3
-              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-              Filter Operator
-                predicate: _col3 is null (type: boolean)
-                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                Select Operator
-                  expressions: _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Group By Operator
-                    keys: _col1 (type: string), _col0 (type: string)
-                    mode: hash
+              Map Join Operator
+                condition map:
+                     Left Outer Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2, _col3
+                residual filter predicates: {(_col1 <> _col2)}
+                Statistics: Num rows: 841 Data size: 254011 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: _col3 is null (type: boolean)
+                  Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string)
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      table:
-                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-          Group By Operator
-            keys: KEY._col0 (type: string)
-            mode: mergepartial
-            outputColumnNames: _col0
-            Statistics: Num rows: 855 Data size: 172430 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: _col0 (type: string), true (type: boolean)
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 855 Data size: 172430 Basic stats: COMPLETE Column stats: NONE
-              Mux Operator
-                Statistics: Num rows: 2566 Data size: 517492 Basic stats: COMPLETE Column stats: NONE
-                Join Operator
-                  condition map:
-                       Left Outer Join 0 to 1
-                  keys:
-                    0 _col1 (type: string)
-                    1 _col0 (type: string)
-                  outputColumnNames: _col0, _col1, _col3
-                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Filter Operator
-                    predicate: _col3 is null (type: boolean)
-                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
+                    Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      keys: _col1 (type: string), _col0 (type: string)
+                      mode: hash
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Group By Operator
-                        keys: _col1 (type: string), _col0 (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                      Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                      File Output Operator
+                        compressed: false
+                        table:
+                            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Local Work:
+        Map Reduce Local Work
 
-  Stage: Stage-16
+  Stage: Stage-14
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2622,22 +2508,22 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+          Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string)
             outputColumnNames: _col1
-            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+            Statistics: Num rows: 210 Data size: 63427 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               keys: _col1 (type: string)
               mode: complete
               outputColumnNames: _col0
-              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+              Statistics: Num rows: 105 Data size: 31713 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
                 table:
@@ -2668,7 +2554,7 @@ STAGE PLANS:
                     0 
                     1 
 
-  Stage: Stage-14
+  Stage: Stage-12
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -2679,25 +2565,25 @@ STAGE PLANS:
                 0 
                 1 
               outputColumnNames: _col0, _col1, _col2
-              Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
               Filter Operator
                 predicate: (_col1 <> _col2) (type: boolean)
-                Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col0 (type: string), _col2 (type: string)
                   outputColumnNames: _col0, _col2
-                  Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                   Group By Operator
                     aggregations: count()
                     keys: _col2 (type: string), _col0 (type: string)
                     mode: hash
                     outputColumnNames: _col0, _col1, _col2
-                    Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
                       key expressions: _col0 (type: string), _col1 (type: string)
                       sort order: ++
                       Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                      Statistics: Num rows: 9 Data size: 1815 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 945 Data size: 475992 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col2 (type: bigint)
       Local Work:
         Map Reduce Local Work
@@ -2707,16 +2593,16 @@ STAGE PLANS:
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2
-          Statistics: Num rows: 4 Data size: 806 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 472 Data size: 237744 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string), _col2 (type: bigint)
             outputColumnNames: _col1, _col2
-            Statistics: Num rows: 4 Data size: 806 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 472 Data size: 237744 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               keys: _col1 (type: string), _col2 (type: bigint)
               mode: hash
               outputColumnNames: _col0, _col1
-              Statistics: Num rows: 4 Data size: 806 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 472 Data size: 237744 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
                 table:
@@ -2724,17 +2610,68 @@ STAGE PLANS:
                     output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                     serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
+  Stage: Stage-13
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: b
+            Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: key (type: string), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+              Reduce Output Operator
+                sort order: 
+                Statistics: Num rows: 29 Data size: 5820 Basic stats: COMPLETE Column stats: NONE
+                value expressions: _col0 (type: string), _col1 (type: string)
+          TableScan
+            Reduce Output Operator
+              sort order: 
+              Statistics: Num rows: 29 Data size: 2910 Basic stats: COMPLETE Column stats: NONE
+              value expressions: _col0 (type: string), _col1 (type: boolean)
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Outer Join 0 to 1
+          keys:
+            0 
+            1 
+          outputColumnNames: _col0, _col1, _col2, _col3
+          residual filter predicates: {(_col1 <> _col2)}
+          Statistics: Num rows: 841 Data size: 254011 Basic stats: COMPLETE Column stats: NONE
+          Filter Operator
+            predicate: _col3 is null (type: boolean)
+            Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: _col0 (type: string), _col1 (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+              Group By Operator
+                keys: _col1 (type: string), _col0 (type: string)
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 420 Data size: 126854 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
   Stage: Stage-0
     Fetch Operator
       limit: -1
       Processor Tree:
         ListSink
 
-Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[186][bigTable=?] in task 'Stage-8:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[187][bigTable=?] in task 'Stage-9:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[188][bigTable=?] in task 'Stage-14:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[189][bigTable=?] in task 'Stage-15:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[146][bigTable=?] in task 'Stage-18:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[147][bigTable=?] in task 'Stage-7:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[148][bigTable=?] in task 'Stage-21:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[32][tables = [$hdt$_2, $hdt$_3]] in Stage 'Stage-8:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[149][bigTable=?] in task 'Stage-12:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[150][bigTable=?] in task 'Stage-24:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[72][tables = [$hdt$_3, $hdt$_4]] in Stage 'Stage-13:MAPRED' is a cross product
 PREHOOK: query: select key, value, count(*)
 from src_null b
 where NOT EXISTS (select key from src_null where src_null.value <> b.value)

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_notexists.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notexists.q.out b/ql/src/test/results/clientpositive/subquery_notexists.q.out
index 039df03..a6175f8 100644
--- a/ql/src/test/results/clientpositive/subquery_notexists.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notexists.q.out
@@ -29,28 +29,28 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (key = key) and (value > 'val_2')) (type: boolean)
-              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((value > 'val_2') and key is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: key (type: string), value (type: string)
                 mode: hash
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                  Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
             outputColumnNames: _col0, _col1, _col2
-            Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
               table:
@@ -78,7 +78,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-              Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col2 (type: boolean)
       Reduce Operator Tree:
         Join Operator
@@ -280,37 +280,37 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (value > 'val_2')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: (value > 'val_2') (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: value (type: string), key (type: string)
                 mode: hash
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string)
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string)
             outputColumnNames: _col1
-            Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               keys: _col1 (type: string)
               mode: complete
               outputColumnNames: _col0
-              Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: _col0 (type: string), true (type: boolean)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   table:
@@ -339,7 +339,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string)
               sort order: +
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: boolean)
       Reduce Operator Tree:
         Join Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notexists_having.q.out b/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
index fda801d..0d90e1b 100644
--- a/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
@@ -72,7 +72,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-              Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col2 (type: boolean)
       Reduce Operator Tree:
         Join Operator
@@ -105,28 +105,28 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (key = key) and (value > 'val_12')) (type: boolean)
-              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((value > 'val_12') and key is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: key (type: string), value (type: string)
                 mode: hash
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                  Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
             outputColumnNames: _col0, _col1, _col2
-            Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
               table:
@@ -255,7 +255,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string)
               sort order: +
               Map-reduce partition columns: _col0 (type: string)
-              Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: boolean)
       Reduce Operator Tree:
         Join Operator
@@ -288,37 +288,37 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (value > 'val_12')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: (value > 'val_12') (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 keys: value (type: string), key (type: string)
                 mode: hash
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string)
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Group By Operator
           keys: KEY._col0 (type: string), KEY._col1 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string)
             outputColumnNames: _col1
-            Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
             Group By Operator
               keys: _col1 (type: string)
               mode: complete
               outputColumnNames: _col0
-              Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: _col0 (type: string), true (type: boolean)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 20 Data size: 212 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   table:

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_notin_having.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.out
index 462dda5..433609d 100644
--- a/ql/src/test/results/clientpositive/subquery_notin_having.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.out
@@ -286,7 +286,7 @@ STAGE PLANS:
               key expressions: _col0 (type: double)
               sort order: +
               Map-reduce partition columns: _col0 (type: double)
-              Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: bigint), _col2 (type: bigint)
       Reduce Operator Tree:
         Join Operator
@@ -319,7 +319,7 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: double)
               sort order: ++
               Map-reduce partition columns: _col0 (type: string), _col1 (type: double)
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col2 (type: boolean)
       Reduce Operator Tree:
         Join Operator
@@ -375,18 +375,18 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1, _col2
           Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((_col1 = _col1) and ((_col2 - _col1) > 600.0)) (type: boolean)
-            Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+            predicate: (((_col2 - _col1) > 600.0) and _col1 is not null) (type: boolean)
+            Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               expressions: _col0 (type: string), _col1 (type: double)
               outputColumnNames: _col0, _col1
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 aggregations: count(), count(_col0)
                 keys: _col1 (type: double)
                 mode: hash
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   table:
@@ -402,7 +402,7 @@ STAGE PLANS:
               key expressions: _col0 (type: double)
               sort order: +
               Map-reduce partition columns: _col0 (type: double)
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: bigint), _col2 (type: bigint)
       Reduce Operator Tree:
         Group By Operator
@@ -410,7 +410,7 @@ STAGE PLANS:
           keys: KEY._col0 (type: double)
           mode: mergepartial
           outputColumnNames: _col0, _col1, _col2
-          Statistics: Num rows: 1 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
             table:
@@ -448,19 +448,19 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1, _col2
           Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((_col1 = _col1) and ((_col2 - _col1) > 600.0)) (type: boolean)
-            Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+            predicate: (((_col2 - _col1) > 600.0) and _col1 is not null) (type: boolean)
+            Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               expressions: _col0 (type: string), _col1 (type: double)
               outputColumnNames: _col0, _col1
-              Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
               Filter Operator
                 predicate: _col0 is not null (type: boolean)
-                Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: double), true (type: boolean)
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
                     table:


[5/6] hive git commit: HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
index 718bc13..a876c62 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
@@ -2216,18 +2216,18 @@ STAGE PLANS:
                   alias: part
                   Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((p_type = p_type) and (p_container = p_container)) (type: boolean)
-                    Statistics: Num rows: 6 Data size: 1902 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: (p_type is not null and p_container is not null) (type: boolean)
+                    Statistics: Num rows: 26 Data size: 8242 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       keys: p_type (type: string), p_name (type: string), p_container (type: string)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
                         sort order: +++
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
-                        Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Map 7 
@@ -2312,12 +2312,12 @@ STAGE PLANS:
                   0 _col4 (type: string), _col1 (type: string), _col6 (type: string)
                   1 _col1 (type: string), _col0 (type: string), _col2 (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
-                Statistics: Num rows: 5 Data size: 3581 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 14 Data size: 4533 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col4 (type: string)
                   sort order: +
                   Map-reduce partition columns: _col4 (type: string)
-                  Statistics: Num rows: 5 Data size: 3581 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 14 Data size: 4533 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
         Reducer 3 
             Execution mode: llap
@@ -2329,12 +2329,12 @@ STAGE PLANS:
                   0 _col4 (type: string)
                   1 _col0 (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col13, _col14
-                Statistics: Num rows: 5 Data size: 3939 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 15 Data size: 4986 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col4 (type: string), _col3 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col4 (type: string), _col3 (type: string)
-                  Statistics: Num rows: 5 Data size: 3939 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 15 Data size: 4986 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col13 (type: bigint), _col14 (type: bigint)
         Reducer 4 
             Execution mode: llap
@@ -2346,17 +2346,17 @@ STAGE PLANS:
                   0 _col4 (type: string), _col3 (type: string)
                   1 _col1 (type: string), _col0 (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col13, _col14, _col17
-                Statistics: Num rows: 5 Data size: 4332 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 16 Data size: 5484 Basic stats: COMPLETE Column stats: NONE
                 Filter Operator
                   predicate: (not CASE WHEN ((_col13 = 0)) THEN (false) WHEN (_col13 is null) THEN (false) WHEN (_col17 is not null) THEN (true) WHEN (_col3 is null) THEN (null) WHEN ((_col14 < _col13)) THEN (true) ELSE (false) END) (type: boolean)
-                  Statistics: Num rows: 3 Data size: 2599 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 8 Data size: 2742 Basic stats: COMPLETE Column stats: NONE
                   Select Operator
                     expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
                     outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
-                    Statistics: Num rows: 3 Data size: 2599 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 8 Data size: 2742 Basic stats: COMPLETE Column stats: NONE
                     File Output Operator
                       compressed: false
-                      Statistics: Num rows: 3 Data size: 2599 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 8 Data size: 2742 Basic stats: COMPLETE Column stats: NONE
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -2368,16 +2368,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string)
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string)
                     sort order: +++
                     Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string)
-                    Statistics: Num rows: 3 Data size: 951 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 13 Data size: 4121 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 8 
             Execution mode: llap
             Reduce Operator Tree:
@@ -2953,15 +2953,15 @@ STAGE PLANS:
                   alias: lineitem
                   Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-                    Statistics: Num rows: 7 Data size: 728 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+                    Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: l_orderkey (type: int), l_linenumber (type: int), l_quantity (type: double)
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 7 Data size: 721 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         sort order: 
-                        Statistics: Num rows: 7 Data size: 721 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: double)
             Execution mode: llap
             LLAP IO: no inputs
@@ -3086,12 +3086,12 @@ STAGE PLANS:
                   0 
                   1 
                 outputColumnNames: _col0, _col1, _col2, _col4, _col5
-                Statistics: Num rows: 7 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 14 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col2 (type: double)
                   sort order: +
                   Map-reduce partition columns: _col2 (type: double)
-                  Statistics: Num rows: 7 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 14 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col0 (type: int), _col1 (type: int), _col4 (type: bigint), _col5 (type: bigint)
         Reducer 8 
             Execution mode: llap
@@ -3103,14 +3103,14 @@ STAGE PLANS:
                   0 _col2 (type: double)
                   1 _col0 (type: double)
                 outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col7
-                Statistics: Num rows: 7 Data size: 252 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 14 Data size: 504 Basic stats: COMPLETE Column stats: COMPLETE
                 Filter Operator
                   predicate: (not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col7 is not null) THEN (true) WHEN (_col2 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END) (type: boolean)
-                  Statistics: Num rows: 4 Data size: 144 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 7 Data size: 252 Basic stats: COMPLETE Column stats: COMPLETE
                   Select Operator
                     expressions: _col0 (type: int), _col1 (type: int)
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       keys: _col0 (type: int), _col1 (type: int)
                       mode: hash

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
index d10fd84..018ef1d 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
@@ -341,23 +341,23 @@ STAGE PLANS:
                       Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
                       value expressions: _col0 (type: string), _col2 (type: int)
                   Filter Operator
-                    predicate: (p_mfgr = p_mfgr) (type: boolean)
-                    Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: p_mfgr is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
                       TopN Hash Memory Usage: 0.1
                       value expressions: p_name (type: string)
                   Filter Operator
-                    predicate: (p_mfgr = p_mfgr) (type: boolean)
-                    Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: p_mfgr is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 2899 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
                       TopN Hash Memory Usage: 0.1
                       value expressions: p_name (type: string)
             Execution mode: llap
@@ -410,7 +410,7 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col1, _col2, _col5
-                Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -431,25 +431,25 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col1 (type: string), _col2 (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         aggregations: count(), count(_col0)
                         keys: _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1, _col2
-                        Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
                           value expressions: _col1 (type: bigint), _col2 (type: bigint)
         Reducer 5 
             Execution mode: llap
@@ -459,12 +459,12 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col0 (type: string)
                   sort order: +
                   Map-reduce partition columns: _col0 (type: string)
-                  Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col1 (type: bigint), _col2 (type: bigint)
         Reducer 6 
             Execution mode: llap
@@ -472,7 +472,7 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col1, _col2, _col5
-                Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -493,24 +493,24 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 6383 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 26 Data size: 12766 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col1 (type: string), _col2 (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 1964 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 3928 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 7 
             Execution mode: llap
             Reduce Operator Tree:
@@ -518,19 +518,19 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
                 Filter Operator
                   predicate: _col0 is not null (type: boolean)
-                  Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
                   Select Operator
                     expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
                     outputColumnNames: _col0, _col1, _col2
-                    Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE
                     Reduce Output Operator
                       key expressions: _col0 (type: string), _col1 (type: string)
                       sort order: ++
                       Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                      Statistics: Num rows: 2 Data size: 446 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE
                       value expressions: _col2 (type: boolean)
 
   Stage: Stage-0
@@ -925,25 +925,25 @@ STAGE PLANS:
                   alias: part
                   Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: (p_mfgr = p_mfgr) (type: boolean)
-                    Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: p_mfgr is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                       TopN Hash Memory Usage: 0.1
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                       TopN Hash Memory Usage: 0.1
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                       TopN Hash Memory Usage: 0.1
             Execution mode: llap
             LLAP IO: no inputs
@@ -953,7 +953,7 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col2, _col5
-                Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -974,25 +974,25 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col2 (type: string), _col5 (type: int)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         aggregations: min(_col1)
                         keys: _col0 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                           value expressions: _col1 (type: int)
         Reducer 11 
             Execution mode: llap
@@ -1002,23 +1002,23 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col1 (type: int), _col0 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: _col0 is not null (type: boolean)
-                    Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col0 (type: int), _col1 (type: string), true (type: boolean)
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col1 (type: string), _col0 (type: int)
                         sort order: ++
                         Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
-                        Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col2 (type: boolean)
         Reducer 2 
             Execution mode: llap
@@ -1092,7 +1092,7 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col2, _col5
-                Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -1113,24 +1113,24 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col2 (type: string)
                       outputColumnNames: _col0
-                      Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: string)
                         mode: hash
                         outputColumnNames: _col0
-                        Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 7 
             Execution mode: llap
             Reduce Operator Tree:
@@ -1138,18 +1138,18 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0
-                Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
                 Group By Operator
                   aggregations: count()
                   keys: _col0 (type: string)
                   mode: complete
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col0 (type: string)
                     sort order: +
                     Map-reduce partition columns: _col0 (type: string)
-                    Statistics: Num rows: 1 Data size: 106 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col1 (type: bigint)
         Reducer 8 
             Execution mode: llap
@@ -1157,7 +1157,7 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col2, _col5
-                Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -1178,25 +1178,25 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col2 (type: string), _col5 (type: int)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         aggregations: min(_col1)
                         keys: _col0 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                           value expressions: _col1 (type: int)
         Reducer 9 
             Execution mode: llap
@@ -1206,18 +1206,18 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                 Group By Operator
                   aggregations: count(), count(_col1)
                   keys: _col0 (type: string)
                   mode: complete
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col0 (type: string)
                     sort order: +
                     Map-reduce partition columns: _col0 (type: string)
-                    Statistics: Num rows: 1 Data size: 114 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col1 (type: bigint), _col2 (type: bigint)
 
   Stage: Stage-0
@@ -3004,30 +3004,30 @@ STAGE PLANS:
                   alias: p
                   Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((p_size = p_size) and (p_partkey = p_partkey)) (type: boolean)
-                    Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: (p_size is not null and p_partkey is not null) (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       aggregations: count(), count(p_name)
                       keys: p_partkey (type: int), p_size (type: int)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2, _col3
-                      Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: int), _col1 (type: int)
                         sort order: ++
                         Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
-                        Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col2 (type: bigint), _col3 (type: bigint)
                     Group By Operator
                       keys: p_partkey (type: int), p_name (type: string), p_size (type: int)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int)
-                        Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
@@ -3080,12 +3080,12 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int), KEY._col1 (type: int)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col0 (type: int), _col1 (type: int)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
-                  Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 13 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col2 (type: bigint), _col3 (type: bigint)
         Reducer 6 
             Execution mode: llap
@@ -3094,23 +3094,23 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int)
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: _col0 is not null (type: boolean)
-                    Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), true (type: boolean)
                       outputColumnNames: _col0, _col1, _col2, _col3
-                      Statistics: Num rows: 3 Data size: 399 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int)
-                        Statistics: Num rows: 3 Data size: 399 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 13 Data size: 1729 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col3 (type: boolean)
 
   Stage: Stage-0

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index ceccc88..3a0d146 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -2300,19 +2300,19 @@ STAGE PLANS:
                   alias: p
                   Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((p_size = p_size) and (p_partkey = p_partkey)) (type: boolean)
-                    Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: (p_size is not null and p_partkey is not null) (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       aggregations: count(p_name)
                       keys: p_partkey (type: int), p_size (type: int)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: int), _col1 (type: int)
                         sort order: ++
                         Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
-                        Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col2 (type: bigint)
             Execution mode: llap
             LLAP IO: no inputs
@@ -2349,16 +2349,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int), KEY._col1 (type: int)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 13 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col2 (type: bigint), true (type: boolean), _col0 (type: int), _col1 (type: int)
                   outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col2 (type: int), _col3 (type: int)
                     sort order: ++
                     Map-reduce partition columns: _col2 (type: int), _col3 (type: int)
-                    Statistics: Num rows: 3 Data size: 60 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col0 (type: bigint), _col1 (type: boolean)
 
   Stage: Stage-0
@@ -3861,12 +3861,12 @@ STAGE PLANS:
                   alias: lineitem
                   Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-                    Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+                    Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: l_orderkey (type: int), l_linenumber (type: int)
                       outputColumnNames: l_orderkey, l_linenumber
-                      Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         aggregations: min(l_orderkey)
                         keys: l_linenumber (type: int)
@@ -4072,12 +4072,12 @@ STAGE PLANS:
                   alias: lineitem
                   Statistics: Num rows: 100 Data size: 9600 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-                    Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+                    Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: l_orderkey (type: int), l_linenumber (type: int)
                       outputColumnNames: l_orderkey, l_linenumber
-                      Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         aggregations: min(l_orderkey)
                         keys: l_linenumber (type: int)

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
index 440e9d7..703d19d 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
@@ -1825,15 +1825,15 @@ STAGE PLANS:
                       Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
                       Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key > '9')) (type: boolean)
-                    Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((key > '9') and value is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: value (type: string)
                       outputColumnNames: _col1
-                      Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         sort order: 
-                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
@@ -1885,17 +1885,17 @@ STAGE PLANS:
                   0 
                   1 
                 outputColumnNames: _col1, _col2
-                Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 166 Data size: 45650 Basic stats: COMPLETE Column stats: COMPLETE
                 Group By Operator
                   keys: _col2 (type: string), _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 36 Data size: 9900 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 72 Data size: 19800 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col0 (type: string), _col1 (type: string)
                     sort order: ++
                     Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                    Statistics: Num rows: 36 Data size: 9900 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 72 Data size: 19800 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 4 
             Execution mode: llap
             Reduce Operator Tree:
@@ -1903,12 +1903,12 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 36 Data size: 9900 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 72 Data size: 19800 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                  Statistics: Num rows: 36 Data size: 9900 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 72 Data size: 19800 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 6 
             Execution mode: llap
             Reduce Operator Tree:
@@ -1998,17 +1998,17 @@ STAGE PLANS:
                       Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
                       Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key > '9')) (type: boolean)
-                    Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((key > '9') and value is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: value (type: string)
                       outputColumnNames: _col1
-                      Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col1 (type: string)
                         sort order: +
                         Map-reduce partition columns: _col1 (type: string)
-                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: value is not null (type: boolean)
                     Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
@@ -2054,21 +2054,21 @@ STAGE PLANS:
                   0 _col1 (type: string)
                   1 _col2 (type: string)
                 outputColumnNames: _col1, _col2, _col3
-                Statistics: Num rows: 83 Data size: 23157 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 166 Data size: 46314 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: CASE WHEN (_col3 is null) THEN (null) ELSE (_col2) END (type: string), _col1 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 166 Data size: 45650 Basic stats: COMPLETE Column stats: COMPLETE
                   Group By Operator
                     keys: _col0 (type: string), _col1 (type: string)
                     mode: hash
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE
                     Reduce Output Operator
                       key expressions: _col0 (type: string), _col1 (type: string)
                       sort order: ++
                       Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                      Statistics: Num rows: 41 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 83 Data size: 22825 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 4 
             Execution mode: llap
             Reduce Operator Tree:

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
index ad199e2..b64e0f4 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
@@ -157,33 +157,33 @@ STAGE PLANS:
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
                         Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key = key) and (value > 'val_11')) (type: boolean)
-                    Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((value > 'val_11') and key is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       aggregations: count(), count(key)
                       keys: key (type: string), value (type: string)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2, _col3
-                      Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: string), _col1 (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                        Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col2 (type: bigint), _col3 (type: bigint)
                   Filter Operator
-                    predicate: ((value = value) and (key = key) and (value > 'val_11')) (type: boolean)
-                    Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((value > 'val_11') and key is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       keys: key (type: string), value (type: string)
                       mode: hash
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: string), _col1 (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                        Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (key < '11') (type: boolean)
                     Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
@@ -197,33 +197,33 @@ STAGE PLANS:
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
                         Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key = key) and (value > 'val_11')) (type: boolean)
-                    Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((value > 'val_11') and key is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       aggregations: count(), count(key)
                       keys: key (type: string), value (type: string)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2, _col3
-                      Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: string), _col1 (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                        Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col2 (type: bigint), _col3 (type: bigint)
                   Filter Operator
-                    predicate: ((value = value) and (key = key) and (value > 'val_11')) (type: boolean)
-                    Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((value > 'val_11') and key is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       keys: key (type: string), value (type: string)
                       mode: hash
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: string), _col1 (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                        Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Map 11 
@@ -260,12 +260,12 @@ STAGE PLANS:
                   0 _col0 (type: string)
                   1 _col0 (type: string)
                 outputColumnNames: _col2, _col3, _col4
-                Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col4 (type: string), _col2 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col4 (type: string), _col2 (type: string)
-                  Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col3 (type: boolean)
         Reducer 12 
             Execution mode: llap
@@ -345,12 +345,12 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                  Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col2 (type: bigint), _col3 (type: bigint)
         Reducer 5 
             Execution mode: llap
@@ -359,16 +359,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
                   outputColumnNames: _col0, _col2, _col3
-                  Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col0 (type: string)
                     sort order: +
                     Map-reduce partition columns: _col0 (type: string)
-                    Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col2 (type: string), _col3 (type: boolean)
         Reducer 6 
             Execution mode: llap
@@ -380,12 +380,12 @@ STAGE PLANS:
                   0 _col0 (type: string)
                   1 _col0 (type: string)
                 outputColumnNames: _col2, _col3, _col4
-                Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col4 (type: string), _col2 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col4 (type: string), _col2 (type: string)
-                  Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col3 (type: boolean)
         Reducer 7 
             Execution mode: llap
@@ -425,12 +425,12 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
                   key expressions: _col0 (type: string), _col1 (type: string)
                   sort order: ++
                   Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                  Statistics: Num rows: 20 Data size: 3880 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
                   value expressions: _col2 (type: bigint), _col3 (type: bigint)
         Reducer 9 
             Execution mode: llap
@@ -439,16 +439,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string), KEY._col1 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
                   outputColumnNames: _col0, _col2, _col3
-                  Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col0 (type: string)
                     sort order: +
                     Map-reduce partition columns: _col0 (type: string)
-                    Statistics: Num rows: 20 Data size: 3640 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col2 (type: string), _col3 (type: boolean)
 
   Stage: Stage-0

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
index 8aabb62..8df5a64 100644
--- a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
@@ -402,9 +402,9 @@ STAGE PLANS:
                     Filter Vectorization:
                         className: VectorFilterOperator
                         native: true
-                        predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, FilterLongColEqualLongColumn(col 3, col 3) -> boolean) -> boolean
-                    predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-                    Statistics: Num rows: 7 Data size: 672 Basic stats: COMPLETE Column stats: COMPLETE
+                        predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, SelectColumnIsNotNull(col 3) -> boolean) -> boolean
+                    predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+                    Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: l_orderkey (type: int), l_linenumber (type: int)
                       outputColumnNames: _col0, _col1
@@ -412,7 +412,7 @@ STAGE PLANS:
                           className: VectorSelectOperator
                           native: true
                           projectedOutputColumns: [0, 3]
-                      Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 14 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         Group By Vectorization:
                             className: VectorGroupByOperator
@@ -425,7 +425,7 @@ STAGE PLANS:
                         keys: _col0 (type: int), _col1 (type: int)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: int), _col1 (type: int)
                           sort order: ++
@@ -434,7 +434,7 @@ STAGE PLANS:
                               className: VectorReduceSinkMultiKeyOperator
                               native: true
                               nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
-                          Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: vectorized, llap
             LLAP IO: no inputs
             Map Vectorization:


[3/6] hive git commit: HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/spark/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index ae4e6fb..1a1689b 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -155,22 +155,22 @@ STAGE PLANS:
                   alias: a
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((value = value) and (key > '9')) (type: boolean)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((key > '9') and value is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
         Reducer 2 
             Reduce Operator Tree:
               Join Operator
@@ -440,13 +440,13 @@ STAGE PLANS:
                   alias: part
                   Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: (p_mfgr = p_mfgr) (type: boolean)
-                    Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                    predicate: p_mfgr is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                       TopN Hash Memory Usage: 0.1
         Reducer 2 
             Reduce Operator Tree:
@@ -474,7 +474,7 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col2, _col5
-                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -495,25 +495,25 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: _col2 (type: string), _col5 (type: int)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         aggregations: min(_col1)
                         keys: _col0 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: string)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                           value expressions: _col1 (type: int)
         Reducer 5 
             Reduce Operator Tree:
@@ -522,16 +522,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col1 (type: int), _col0 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col1 (type: string), _col0 (type: int)
                     sort order: ++
                     Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
-                    Statistics: Num rows: 2 Data size: 242 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
 
   Stage: Stage-0
     Fetch Operator
@@ -563,21 +563,21 @@ Manufacturer#2	almond aquamarine midnight light salmon	2
 Manufacturer#3	almond antique misty red olive	1
 Manufacturer#4	almond aquamarine yellow dodger mint	7
 Manufacturer#5	almond antique sky peru orange	2
-PREHOOK: query: explain 
-select * 
-from src b 
+PREHOOK: query: explain
+select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 PREHOOK: type: QUERY
-POSTHOOK: query: explain 
-select * 
-from src b 
+POSTHOOK: query: explain
+select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 POSTHOOK: type: QUERY
@@ -612,22 +612,22 @@ STAGE PLANS:
                   alias: a
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((value = value) and (key > '9')) (type: boolean)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((key > '9') and value is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
         Reducer 2 
             Reduce Operator Tree:
               Join Operator
@@ -652,21 +652,21 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-PREHOOK: query: select * 
-from src b 
+PREHOOK: query: select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 #### A masked pattern was here ####
-POSTHOOK: query: select * 
-from src b 
+POSTHOOK: query: select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 POSTHOOK: type: QUERY
@@ -683,6 +683,143 @@ POSTHOOK: Input: default@src
 97	val_97
 98	val_98
 98	val_98
+PREHOOK: query: explain
+select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Map 3 (GROUP, 2)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: key (type: string), value (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: string)
+                      Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+                      value expressions: _col1 (type: string)
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: a
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key > '9') (type: boolean)
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      keys: key (type: string)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col0 (type: string)
+                  1 _col0 (type: string)
+                outputColumnNames: _col0, _col1, _col3
+                Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (_col1 <> _col3) (type: boolean)
+                  Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 4 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: string)
+                  sort order: +
+                  Map-reduce partition columns: _col0 (type: string)
+                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+90	val_90
+90	val_90
+90	val_90
+92	val_92
+95	val_95
+95	val_95
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
 PREHOOK: query: select p_mfgr, p_name, p_size 
 from part 
 where part.p_size in 
@@ -1864,18 +2001,18 @@ STAGE PLANS:
                   alias: p
                   Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((p_size = p_size) and (p_partkey = p_partkey)) (type: boolean)
-                    Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE
+                    predicate: (p_size is not null and p_partkey is not null) (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                     Group By Operator
                       keys: p_partkey (type: int), p_name (type: string), p_size (type: int)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                       Reduce Output Operator
                         key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int)
-                        Statistics: Num rows: 6 Data size: 726 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
         Reducer 2 
             Reduce Operator Tree:
               Join Operator
@@ -1899,16 +2036,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int)
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int)
                     sort order: +++
                     Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int)
-                    Statistics: Num rows: 3 Data size: 363 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
 
   Stage: Stage-0
     Fetch Operator
@@ -3102,22 +3239,22 @@ STAGE PLANS:
                   alias: part
                   Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((p_size < 10) and (p_mfgr = p_mfgr)) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((p_size < 10) and p_mfgr is not null) (type: boolean)
+                    Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: p_mfgr (type: string), p_name (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 4 Data size: 484 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 8 Data size: 968 Basic stats: COMPLETE Column stats: NONE
         Reducer 2 
             Reduce Operator Tree:
               Join Operator
@@ -4999,3 +5136,373 @@ POSTHOOK: query: drop table tt
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@tt
 POSTHOOK: Output: default@tt
+Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product
+PREHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Reducer 7 (PARTITION-LEVEL SORT, 1)
+        Reducer 5 <- Reducer 4 (GROUP, 2)
+        Reducer 7 <- Map 6 (GROUP, 2)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string)
+                    outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: _col1 (type: string), _col5 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col1 (type: string), _col5 (type: int)
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string)
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: p
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: p_type (type: string), p_size (type: int)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Reduce Output Operator
+                      sort order: 
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      value expressions: _col0 (type: string), _col1 (type: int)
+        Map 6 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: p_name (type: string)
+                    outputColumnNames: p_name
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      keys: p_name (type: string)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col1 (type: string), _col5 (type: int)
+                  1 _col1 (type: string), _col0 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                Statistics: Num rows: 185 Data size: 45180 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 185 Data size: 45180 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 4 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (_col0 <> _col2) (type: boolean)
+                  Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col1 (type: int), _col2 (type: string)
+                    outputColumnNames: _col1, _col2
+                    Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: max(_col1)
+                      keys: _col2 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: int)
+        Reducer 5 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: max(VALUE._col0)
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 169 Data size: 41073 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col1 (type: int), _col0 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 169 Data size: 41073 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col1 (type: string), _col0 (type: int)
+                    sort order: ++
+                    Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
+                    Statistics: Num rows: 169 Data size: 41073 Basic stats: COMPLETE Column stats: NONE
+        Reducer 7 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                  value expressions: _col0 (type: string)
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+Warning: Shuffle Join JOIN[12][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product
+PREHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+15103	almond aquamarine dodger light gainsboro	Manufacturer#5	Brand#53	ECONOMY BURNISHED STEEL	46	LG PACK	1018.1	packages hinder carefu
+Warning: Shuffle Join JOIN[16][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product
+PREHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2)
+        Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Reducer 8 (PARTITION-LEVEL SORT, 1)
+        Reducer 5 <- Map 9 (PARTITION-LEVEL SORT, 2), Reducer 4 (PARTITION-LEVEL SORT, 2)
+        Reducer 6 <- Reducer 5 (GROUP, 2)
+        Reducer 8 <- Map 7 (GROUP, 2)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string)
+                    outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: _col4 (type: string), _col5 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col4 (type: string), _col5 (type: int)
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string)
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: p
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: p_type is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: p_name (type: string), p_type (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: string), _col1 (type: string)
+        Map 7 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: p_type (type: string)
+                    outputColumnNames: p_type
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      keys: p_type (type: string)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+        Map 9 
+            Map Operator Tree:
+                TableScan
+                  alias: pp
+                  Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: p_type is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: p_type (type: string), p_size (type: int)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: int)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col4 (type: string), _col5 (type: int)
+                  1 _col1 (type: string), _col0 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                Statistics: Num rows: 203 Data size: 49563 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 203 Data size: 49563 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 4 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (_col2 <> _col0) (type: boolean)
+                  Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col1 (type: string)
+                    sort order: +
+                    Map-reduce partition columns: _col1 (type: string)
+                    Statistics: Num rows: 338 Data size: 82147 Basic stats: COMPLETE Column stats: NONE
+                    value expressions: _col2 (type: string)
+        Reducer 5 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col1 (type: string)
+                  1 _col0 (type: string)
+                outputColumnNames: _col2, _col4
+                Statistics: Num rows: 371 Data size: 90361 Basic stats: COMPLETE Column stats: NONE
+                Group By Operator
+                  keys: _col2 (type: string), _col4 (type: int)
+                  mode: hash
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 371 Data size: 90361 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string), _col1 (type: int)
+                    sort order: ++
+                    Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+                    Statistics: Num rows: 371 Data size: 90361 Basic stats: COMPLETE Column stats: NONE
+        Reducer 6 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string), KEY._col1 (type: int)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 185 Data size: 45058 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col1 (type: int), _col0 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 185 Data size: 45058 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col1 (type: string), _col0 (type: int)
+                    sort order: ++
+                    Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
+                    Statistics: Num rows: 185 Data size: 45058 Basic stats: COMPLETE Column stats: NONE
+        Reducer 8 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
+                  value expressions: _col0 (type: string)
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+Warning: Shuffle Join JOIN[16][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product
+PREHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+105685	almond antique violet chocolate turquoise	Manufacturer#2	Brand#22	MEDIUM ANODIZED COPPER	14	MED CAN	1690.68	ly pending requ
+110592	almond antique salmon chartreuse burlywood	Manufacturer#1	Brand#15	PROMO BURNISHED NICKEL	6	JUMBO PKG	1602.59	 to the furiously
+112398	almond antique metallic orange dim	Manufacturer#3	Brand#32	MEDIUM BURNISHED BRASS	19	JUMBO JAR	1410.39	ole car
+121152	almond antique burnished rose metallic	Manufacturer#1	Brand#14	PROMO PLATED TIN	2	JUMBO BOX	1173.15	e pinto beans h
+121152	almond antique burnished rose metallic	Manufacturer#1	Brand#14	PROMO PLATED TIN	2	JUMBO BOX	1173.15	e pinto beans h
+132666	almond aquamarine rose maroon antique	Manufacturer#2	Brand#24	SMALL POLISHED NICKEL	25	MED BOX	1698.66	even 
+144293	almond antique olive coral navajo	Manufacturer#3	Brand#34	STANDARD POLISHED STEEL	45	JUMBO CAN	1337.29	ag furiously about 
+146985	almond aquamarine midnight light salmon	Manufacturer#2	Brand#23	MEDIUM BURNISHED COPPER	2	SM CASE	2031.98	s cajole caref
+15103	almond aquamarine dodger light gainsboro	Manufacturer#5	Brand#53	ECONOMY BURNISHED STEEL	46	LG PACK	1018.1	packages hinder carefu
+155733	almond antique sky peru orange	Manufacturer#5	Brand#53	SMALL PLATED BRASS	2	WRAP DRUM	1788.73	furiously. bra
+17273	almond antique forest lavender goldenrod	Manufacturer#3	Brand#35	PROMO ANODIZED TIN	14	JUMBO CASE	1190.27	along the
+17927	almond aquamarine yellow dodger mint	Manufacturer#4	Brand#41	ECONOMY BRUSHED COPPER	7	SM PKG	1844.92	ites. eve
+191709	almond antique violet turquoise frosted	Manufacturer#2	Brand#22	ECONOMY POLISHED STEEL	40	MED BOX	1800.7	 haggle
+192697	almond antique blue firebrick mint	Manufacturer#5	Brand#52	MEDIUM BURNISHED TIN	31	LG DRUM	1789.69	ickly ir
+195606	almond aquamarine sandy cyan gainsboro	Manufacturer#2	Brand#25	STANDARD PLATED TIN	18	SM PKG	1701.6	ic de
+33357	almond azure aquamarine papaya violet	Manufacturer#4	Brand#41	STANDARD ANODIZED TIN	12	WRAP CASE	1290.35	reful
+40982	almond antique misty red olive	Manufacturer#3	Brand#32	ECONOMY PLATED COPPER	1	LG PKG	1922.98	c foxes can s
+42669	almond antique medium spring khaki	Manufacturer#5	Brand#51	STANDARD BURNISHED TIN	6	MED CAN	1611.66	sits haggl
+45261	almond aquamarine floral ivory bisque	Manufacturer#4	Brand#42	SMALL PLATED STEEL	27	WRAP CASE	1206.26	careful
+48427	almond antique violet mint lemon	Manufacturer#4	Brand#42	PROMO POLISHED STEEL	39	SM CASE	1375.42	hely ironic i
+49671	almond antique gainsboro frosted violet	Manufacturer#4	Brand#41	SMALL BRUSHED BRASS	10	SM BOX	1620.67	ccounts run quick
+65667	almond aquamarine pink moccasin thistle	Manufacturer#1	Brand#12	LARGE BURNISHED STEEL	42	JUMBO CASE	1632.66	e across the expr
+78486	almond azure blanched chiffon midnight	Manufacturer#5	Brand#52	LARGE BRUSHED BRASS	23	MED BAG	1464.48	hely blith
+85768	almond antique chartreuse lavender yellow	Manufacturer#1	Brand#12	LARGE BRUSHED STEEL	34	SM BAG	1753.76	refull
+86428	almond aquamarine burnished black steel	Manufacturer#1	Brand#12	STANDARD ANODIZED STEEL	28	WRAP BAG	1414.42	arefully 
+90681	almond antique chartreuse khaki white	Manufacturer#3	Brand#31	MEDIUM BURNISHED TIN	17	SM CASE	1671.68	are slyly after the sl

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
index 433b9a2..558a2d0 100644
--- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
@@ -354,9 +354,9 @@ STAGE PLANS:
                     Filter Vectorization:
                         className: VectorFilterOperator
                         native: true
-                        predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, FilterLongColEqualLongColumn(col 3, col 3) -> boolean) -> boolean
-                    predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-                    Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                        predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14, val AIR) -> boolean, SelectColumnIsNotNull(col 3) -> boolean) -> boolean
+                    predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+                    Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: l_orderkey (type: int), l_linenumber (type: int)
                       outputColumnNames: _col0, _col1
@@ -364,7 +364,7 @@ STAGE PLANS:
                           className: VectorSelectOperator
                           native: true
                           projectedOutputColumns: [0, 3]
-                      Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                       Group By Operator
                         Group By Vectorization:
                             className: VectorGroupByOperator
@@ -377,7 +377,7 @@ STAGE PLANS:
                         keys: _col0 (type: int), _col1 (type: int)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                         Spark HashTable Sink Operator
                           Spark Hash Table Sink Vectorization:
                               className: VectorSparkHashTableSinkOperator

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_exists.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_exists.q.out b/ql/src/test/results/clientpositive/subquery_exists.q.out
index cfc7652..f19d8a9 100644
--- a/ql/src/test/results/clientpositive/subquery_exists.q.out
+++ b/ql/src/test/results/clientpositive/subquery_exists.q.out
@@ -40,22 +40,22 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((value = value) and (key = key) and (value > 'val_9')) (type: boolean)
-              Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((value > 'val_9') and key is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string), _col1 (type: string)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: string), _col1 (type: string)
                     sort order: ++
                     Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                    Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/subquery_exists_having.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_exists_having.q.out b/ql/src/test/results/clientpositive/subquery_exists_having.q.out
index 2c41ff6..f9d347d 100644
--- a/ql/src/test/results/clientpositive/subquery_exists_having.q.out
+++ b/ql/src/test/results/clientpositive/subquery_exists_having.q.out
@@ -74,22 +74,22 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((key = key) and (value > 'val_9')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((value > 'val_9') and key is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string)
                 outputColumnNames: _col0
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string)
                   mode: hash
                   outputColumnNames: _col0
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: string)
                     sort order: +
                     Map-reduce partition columns: _col0 (type: string)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
@@ -192,33 +192,33 @@ STAGE PLANS:
             alias: a
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((key = key) and (value > 'val_9')) (type: boolean)
-              Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((value > 'val_9') and key is not null) (type: boolean)
+              Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string)
                 outputColumnNames: _col0
-                Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: string)
                   mode: hash
                   outputColumnNames: _col0
-                  Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: string)
                     sort order: +
                     Map-reduce partition columns: _col0 (type: string)
-                    Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Demux Operator
-          Statistics: Num rows: 583 Data size: 6193 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE
           Group By Operator
             aggregations: count(VALUE._col0)
             keys: KEY._col0 (type: string)
             mode: mergepartial
             outputColumnNames: _col0, _col1
-            Statistics: Num rows: 291 Data size: 3091 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
             Mux Operator
-              Statistics: Num rows: 874 Data size: 9284 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 999 Data size: 10612 Basic stats: COMPLETE Column stats: NONE
               Join Operator
                 condition map:
                      Left Semi Join 0 to 1
@@ -235,7 +235,7 @@ STAGE PLANS:
                       output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                       serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           Mux Operator
-            Statistics: Num rows: 874 Data size: 9284 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 999 Data size: 10612 Basic stats: COMPLETE Column stats: NONE
             Join Operator
               condition map:
                    Left Semi Join 0 to 1


[6/6] hive git commit: HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Posted by ha...@apache.org.
HIVE-16838 : Improve plans for subqueries with non-equi co-related predicates (Vineet Garg via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <ha...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3192d06d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3192d06d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3192d06d

Branch: refs/heads/master
Commit: 3192d06d6b6040e14e68d7b71016398a28de73dc
Parents: 99c0caf
Author: Vineet Garg <vg...@apache.com>
Authored: Tue Jun 6 18:23:00 2017 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jun 28 12:11:16 2017 -0700

----------------------------------------------------------------------
 .../calcite/reloperators/HiveAggregate.java     |  10 +
 .../calcite/reloperators/HiveIntersect.java     |  10 +
 .../calcite/reloperators/HiveUnion.java         |   9 +
 .../calcite/rules/HiveRelDecorrelator.java      | 300 +++++--
 .../test/queries/clientpositive/subquery_in.q   |  50 +-
 .../clientpositive/constprog_partitioner.q.out  |  10 +-
 .../clientpositive/llap/explainuser_1.q.out     |  76 +-
 .../clientpositive/llap/subquery_exists.q.out   |  10 +-
 .../clientpositive/llap/subquery_in.q.out       | 629 ++++++++++++--
 .../clientpositive/llap/subquery_multi.q.out    |  48 +-
 .../clientpositive/llap/subquery_notin.q.out    | 142 ++--
 .../clientpositive/llap/subquery_scalar.q.out   |  26 +-
 .../clientpositive/llap/subquery_select.q.out   |  34 +-
 .../clientpositive/llap/subquery_views.q.out    |  60 +-
 .../llap/vector_mapjoin_reduce.q.out            |  12 +-
 .../results/clientpositive/perf/query16.q.out   | 263 +++---
 .../results/clientpositive/perf/query94.q.out   | 263 +++---
 .../spark/constprog_partitioner.q.out           |  10 +-
 .../clientpositive/spark/subquery_exists.q.out  |  10 +-
 .../clientpositive/spark/subquery_in.q.out      | 611 ++++++++++++--
 .../spark/vector_mapjoin_reduce.q.out           |  10 +-
 .../clientpositive/subquery_exists.q.out        |  10 +-
 .../clientpositive/subquery_exists_having.q.out |  28 +-
 .../clientpositive/subquery_in_having.q.out     | 831 +++++++++----------
 .../clientpositive/subquery_notexists.q.out     |  32 +-
 .../subquery_notexists_having.q.out             |  32 +-
 .../clientpositive/subquery_notin_having.q.out  |  26 +-
 .../subquery_unqualcolumnrefs.q.out             |  82 +-
 .../clientpositive/vector_mapjoin_reduce.q.out  |   8 +-
 29 files changed, 2377 insertions(+), 1265 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
index 63bbdac..e4c0cc5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
@@ -25,6 +25,7 @@ import org.apache.calcite.linq4j.Ord;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
 import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.core.AggregateCall;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
@@ -34,6 +35,7 @@ import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttle;
 
 import com.google.common.collect.Sets;
 
@@ -130,4 +132,12 @@ public class HiveAggregate extends Aggregate implements HiveRelNode {
     return this.aggregateColumnsOrder;
   }
 
+  //required for HiveRelDecorrelator
+  @Override public RelNode accept(RelShuttle shuttle) {
+    if(shuttle instanceof HiveRelShuttle) {
+      return ((HiveRelShuttle)shuttle).visit(this);
+    }
+    return shuttle.visit(this);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveIntersect.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveIntersect.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveIntersect.java
index 19e1e02..60252b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveIntersect.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveIntersect.java
@@ -22,8 +22,10 @@ import java.util.List;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
 import org.apache.calcite.rel.core.Intersect;
 import org.apache.calcite.rel.core.SetOp;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttle;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode.Implementor;
 
 public class HiveIntersect extends Intersect {
@@ -40,4 +42,12 @@ public class HiveIntersect extends Intersect {
   public void implement(Implementor implementor) {
   }
 
+  //required for HiveRelDecorrelator
+  public RelNode accept(RelShuttle shuttle) {
+    if (shuttle instanceof HiveRelShuttle) {
+      return ((HiveRelShuttle)shuttle).visit(this);
+    }
+    return shuttle.visit(this);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java
index 7cfb007..a6b1991 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveUnion.java
@@ -22,8 +22,10 @@ import java.util.List;
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelShuttle;
 import org.apache.calcite.rel.core.SetOp;
 import org.apache.calcite.rel.core.Union;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttle;
 
 public class HiveUnion extends Union implements HiveRelNode {
 
@@ -39,5 +41,12 @@ public class HiveUnion extends Union implements HiveRelNode {
   @Override
   public void implement(Implementor implementor) {
   }
+  //required for HiveRelDecorrelator
+  public RelNode accept(RelShuttle shuttle) {
+    if (shuttle instanceof HiveRelShuttle) {
+      return ((HiveRelShuttle)shuttle).visit(this);
+    }
+    return shuttle.visit(this);
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
index 4c99932..ef50d9d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
@@ -44,11 +44,12 @@ import org.apache.calcite.rel.core.Values;
 import org.apache.calcite.rel.logical.LogicalAggregate;
 import org.apache.calcite.rel.logical.LogicalCorrelate;
 import org.apache.calcite.rel.logical.LogicalFilter;
+import org.apache.calcite.rel.logical.LogicalIntersect;
 import org.apache.calcite.rel.logical.LogicalJoin;
 import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalUnion;
 import org.apache.calcite.rel.metadata.RelMdUtil;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.rules.FilterCorrelateRule;
 import org.apache.calcite.rel.rules.FilterJoinRule;
 import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
 import org.apache.calcite.rel.type.RelDataType;
@@ -61,6 +62,7 @@ import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexShuttle;
 import org.apache.calcite.rex.RexSubQuery;
 import org.apache.calcite.rex.RexUtil;
@@ -85,10 +87,11 @@ import org.apache.calcite.util.Util;
 import org.apache.calcite.util.mapping.Mappings;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -119,6 +122,7 @@ import java.util.NavigableMap;
 import java.util.Objects;
 import java.util.Set;
 import java.util.SortedMap;
+import java.util.Stack;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import javax.annotation.Nonnull;
@@ -177,6 +181,8 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
 
   private final HashSet<LogicalCorrelate> generatedCorRels = Sets.newHashSet();
 
+  private Stack valueGen = new Stack();
+
   //~ Constructors -----------------------------------------------------------
 
   private HiveRelDecorrelator (
@@ -263,6 +269,8 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
       return planner2.findBestExp();
     }
 
+    assert(valueGen.isEmpty());
+
     return root;
   }
 
@@ -320,8 +328,14 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
     return planner.findBestExp();
   }
 
+  protected RexNode decorrelateExpr(RexNode exp, boolean valueGenerator) {
+    DecorrelateRexShuttle shuttle = new DecorrelateRexShuttle();
+    shuttle.setValueGenerator(valueGenerator);
+    return exp.accept(shuttle);
+  }
   protected RexNode decorrelateExpr(RexNode exp) {
     DecorrelateRexShuttle shuttle = new DecorrelateRexShuttle();
+    shuttle.setValueGenerator(false);
     return exp.accept(shuttle);
   }
 
@@ -1107,7 +1121,11 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
         try {
           findCorrelationEquivalent(correlation, ((Filter) rel).getCondition());
         } catch (Util.FoundOne e) {
-          map.put(def, (Integer) e.getNode());
+          // we need to keep predicate kind e.g. EQUAL or NOT EQUAL
+          // so that later while decorrelating LogicalCorrelate appropriate join predicate
+          // is generated
+          def.setPredicateKind((SqlKind)((Pair)e.getNode()).getValue());
+          map.put(def, (Integer)((Pair) e.getNode()).getKey());
         }
       }
       // If all correlation variables are now satisfied, skip creating a value
@@ -1146,16 +1164,22 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
   private void findCorrelationEquivalent(CorRef correlation, RexNode e)
           throws Util.FoundOne {
     switch (e.getKind()) {
-      case EQUALS:
+    // for now only EQUAL and NOT EQUAL corr predicates are optimized
+    case NOT_EQUALS:
+      if((boolean)valueGen.peek()) {
+        // we will need value generator
+        break;
+      }
+    case EQUALS:
         final RexCall call = (RexCall) e;
         final List<RexNode> operands = call.getOperands();
         if (references(operands.get(0), correlation)
                 && operands.get(1) instanceof RexInputRef) {
-          throw new Util.FoundOne(((RexInputRef) operands.get(1)).getIndex());
+          throw new Util.FoundOne(Pair.of(((RexInputRef) operands.get(1)).getIndex(), e.getKind()));
         }
         if (references(operands.get(1), correlation)
                 && operands.get(0) instanceof RexInputRef) {
-          throw new Util.FoundOne(((RexInputRef) operands.get(0)).getIndex());
+          throw new Util.FoundOne(Pair.of(((RexInputRef) operands.get(0)).getIndex(), e.getKind()));
         }
         break;
       case AND:
@@ -1224,17 +1248,22 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
         return null;
       }
 
+      Frame oldInputFrame = frame;
       // If this LogicalFilter has correlated reference, create value generator
       // and produce the correlated variables in the new output.
       if (cm.mapRefRelToCorRef.containsKey(rel)) {
         frame = decorrelateInputWithValueGenerator(rel);
       }
 
-      // Replace the filter expression to reference output of the join
-      // Map filter to the new filter over join
-      relBuilder.push(frame.r).filter(
-          simplifyComparison(decorrelateExpr(rel.getCondition())));
-
+      boolean valueGenerator = true;
+      if(frame.r == oldInputFrame.r) {
+        // this means correated value generator wasn't generated
+        valueGenerator = false;
+      }
+        // Replace the filter expression to reference output of the join
+        // Map filter to the new filter over join
+        relBuilder.push(frame.r).filter(
+            (decorrelateExpr(rel.getCondition(), valueGenerator)));
       // Filter does not change the input ordering.
       // Filter rel does not permute the input.
       // All corvars produced by filter will have the same output positions in the
@@ -1244,39 +1273,6 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
     }
   }
 
-  private RexNode simplifyComparison(RexNode op) {
-    switch(op.getKind()) {
-    case EQUALS:
-    case GREATER_THAN:
-    case GREATER_THAN_OR_EQUAL:
-    case LESS_THAN:
-    case LESS_THAN_OR_EQUAL:
-    case NOT_EQUALS:
-      RexCall e = (RexCall) op;
-      final List<RexNode> operands = new ArrayList<>(e.operands);
-
-      // Simplify "x <op> x"
-      final RexNode o0 = operands.get(0);
-      final RexNode o1 = operands.get(1);
-      // this should only be called when we are creating filter (decorrelate filter)
-      // since in that case null/unknown is treated as false we don't care about
-      // nullability of operands and will always rewrite op=op to op is not null
-      if (RexUtil.eq(o0, o1) )
-        switch (e.getKind()) {
-        case EQUALS:
-        case GREATER_THAN_OR_EQUAL:
-        case LESS_THAN_OR_EQUAL:
-          // "x = x" simplifies to "x is not null" (similarly <= and >=)
-          return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, o0);
-        default:
-          // "x != x" simplifies to "false" (similarly < and >)
-          return rexBuilder.makeLiteral(false);
-        }
-    }
-    return op;
-  }
-
-
     /**
      * Rewrite LogicalFilter.
      *
@@ -1313,9 +1309,15 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
 
     }
 
+    boolean valueGenerator = true;
+    if(frame.r == oldInput) {
+      // this means correated value generator wasn't generated
+      valueGenerator = false;
+    }
+
     // Replace the filter expression to reference output of the join
     // Map filter to the new filter over join
-    relBuilder.push(frame.r).filter(decorrelateExpr(rel.getCondition()));
+    relBuilder.push(frame.r).filter(decorrelateExpr(rel.getCondition(), valueGenerator));
 
 
     // Filter does not change the input ordering.
@@ -1345,6 +1347,9 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
     final RelNode oldLeft = rel.getInput(0);
     final RelNode oldRight = rel.getInput(1);
 
+    boolean mightRequireValueGen = new findIfValueGenRequired().traverse(oldRight);
+    valueGen.push(mightRequireValueGen);
+
     final Frame leftFrame = getInvoke(oldLeft, rel);
     final Frame rightFrame = getInvoke(oldRight, rel);
 
@@ -1381,11 +1386,24 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
       }
       final int newLeftPos = leftFrame.oldToNewOutputs.get(corDef.field);
       final int newRightPos = rightOutput.getValue();
-      conditions.add(
-              rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
-                      RexInputRef.of(newLeftPos, newLeftOutput),
-                      new RexInputRef(newLeftFieldCount + newRightPos,
-                              newRightOutput.get(newRightPos).getType())));
+      if(corDef.getPredicateKind() == SqlKind.NOT_EQUALS) {
+        conditions.add(
+            rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS,
+                RexInputRef.of(newLeftPos, newLeftOutput),
+                new RexInputRef(newLeftFieldCount + newRightPos,
+                    newRightOutput.get(newRightPos).getType())));
+
+      }
+      else {
+        assert(corDef.getPredicateKind() == null
+          || corDef.getPredicateKind() == SqlKind.EQUALS);
+        conditions.add(
+            rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+                RexInputRef.of(newLeftPos, newLeftOutput),
+                new RexInputRef(newLeftFieldCount + newRightPos,
+                    newRightOutput.get(newRightPos).getType())));
+
+      }
 
       // remove this cor var from output position mapping
       corDefOutputs.remove(corDef);
@@ -1428,6 +1446,8 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
             LogicalJoin.create(leftFrame.r, rightFrame.r, condition,
                     ImmutableSet.<CorrelationId>of(), rel.getJoinType().toJoinType());
 
+    valueGen.pop();
+
     return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs);
   }
 
@@ -1820,7 +1840,66 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
 
   /** Shuttle that decorrelates. */
   private class DecorrelateRexShuttle extends RexShuttle {
+    private boolean valueGenerator;
+    public void setValueGenerator(boolean valueGenerator) {
+      this.valueGenerator = valueGenerator;
+    }
+
+    // DecorrelateRexShuttle ends up decorrelating expressions cor.col1 <> $4
+    // to $4=$4 if value generator is not generated, $4<>$4 is further simplified
+    // to false. This is wrong and messes up the whole tree. To prevent this visitCall
+    // is overridden to rewrite/simply such predicates to is not null.
+    // we also need to take care that we do this only for correlated predicates and
+    // not user specified explicit predicates
+    // TODO:  This code should be removed once CALCITE-1851 is fixed and
+    // there is support of not equal
+    @Override  public RexNode visitCall(final RexCall call) {
+      if(!valueGenerator) {
+        switch (call.getKind()) {
+        case EQUALS:
+        case NOT_EQUALS:
+          final List<RexNode> operands = new ArrayList<>(call.operands);
+          RexNode o0 = operands.get(0);
+          RexNode o1 = operands.get(1);
+          boolean isCorrelated = false;
+          if (o0 instanceof RexFieldAccess && (cm.mapFieldAccessToCorRef.get(o0) != null)) {
+            o0 = decorrFieldAccess((RexFieldAccess) o0);
+            isCorrelated = true;
+
+          }
+          if (o1 instanceof RexFieldAccess && (cm.mapFieldAccessToCorRef.get(o1) != null)) {
+            o1 = decorrFieldAccess((RexFieldAccess) o1);
+            isCorrelated = true;
+          }
+          if (isCorrelated && RexUtil.eq(o0, o1)) {
+            return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, o0);
+          }
+
+          final List<RexNode> newOperands = new ArrayList<>();
+          newOperands.add(o0);
+          newOperands.add(o1);
+          boolean[] update = { false };
+          List<RexNode> clonedOperands = visitList(newOperands, update);
+
+          return relBuilder.call(call.getOperator(), clonedOperands);
+        }
+      }
+      return super.visitCall(call);
+    }
+
     @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) {
+      return decorrFieldAccess(fieldAccess);
+    }
+
+    @Override public RexNode visitInputRef(RexInputRef inputRef) {
+      final RexInputRef ref = getNewForOldInputRef(inputRef);
+      if (ref.getIndex() == inputRef.getIndex()
+              && ref.getType() == inputRef.getType()) {
+        return inputRef; // re-use old object, to prevent needless expr cloning
+      }
+      return ref;
+    }
+    private RexNode decorrFieldAccess(RexFieldAccess fieldAccess) {
       int newInputOutputOffset = 0;
       for (RelNode input : currentRel.getInputs()) {
         final Frame frame = map.get(input);
@@ -1835,7 +1914,7 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
               // This input rel does produce the cor var referenced.
               // Assume fieldAccess has the correct type info.
               return new RexInputRef(newInputPos + newInputOutputOffset,
-                      frame.r.getRowType().getFieldList().get(newInputPos)
+                  frame.r.getRowType().getFieldList().get(newInputPos)
                       .getType());
             }
           }
@@ -1849,15 +1928,6 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
       }
       return fieldAccess;
     }
-
-    @Override public RexNode visitInputRef(RexInputRef inputRef) {
-      final RexInputRef ref = getNewForOldInputRef(inputRef);
-      if (ref.getIndex() == inputRef.getIndex()
-              && ref.getType() == inputRef.getType()) {
-        return inputRef; // re-use old object, to prevent needless expr cloning
-      }
-      return ref;
-    }
   }
 
   /** Shuttle that removes correlations. */
@@ -2882,10 +2952,12 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
   static class CorDef implements Comparable<CorDef> {
     public final CorrelationId corr;
     public final int field;
+    private SqlKind predicateKind;
 
     CorDef(CorrelationId corr, int field) {
       this.corr = corr;
       this.field = field;
+      this.predicateKind = null;
     }
 
     @Override public String toString() {
@@ -2910,6 +2982,13 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
       }
       return Integer.compare(field, o.field);
     }
+    public SqlKind getPredicateKind() {
+      return predicateKind;
+    }
+    public void setPredicateKind(SqlKind predKind) {
+      this.predicateKind = predKind;
+
+    }
   }
 
   /** A map of the locations of
@@ -2987,6 +3066,107 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
     }
   }
 
+  private static class findIfValueGenRequired extends HiveRelShuttleImpl {
+    private boolean mightRequireValueGen ;
+    findIfValueGenRequired() { this.mightRequireValueGen = true; }
+
+    private boolean hasRexOver(List<RexNode> projects) {
+      for(RexNode expr : projects) {
+        if(expr instanceof  RexOver) {
+          return true;
+        }
+      }
+      return false;
+    }
+    @Override public RelNode visit(HiveJoin rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+    public RelNode visit(HiveSortLimit rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+    public RelNode visit(HiveUnion rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+    public RelNode visit(LogicalUnion rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+    public RelNode visit(LogicalIntersect rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+
+    public RelNode visit(HiveIntersect rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+
+    @Override public RelNode visit(LogicalJoin rel) {
+      mightRequireValueGen = true;
+      return rel;
+    }
+    @Override public RelNode visit(HiveProject rel) {
+      if(!(hasRexOver(((HiveProject)rel).getProjects()))) {
+        mightRequireValueGen = false;
+        return super.visit(rel);
+      }
+      else {
+        mightRequireValueGen = true;
+        return rel;
+      }
+    }
+    @Override public RelNode visit(LogicalProject rel) {
+      if(!(hasRexOver(((LogicalProject)rel).getProjects()))) {
+        mightRequireValueGen = false;
+        return super.visit(rel);
+      }
+      else {
+        mightRequireValueGen = true;
+        return rel;
+      }
+    }
+    @Override public RelNode visit(HiveAggregate rel) {
+      // if there are aggregate functions or grouping sets we will need
+      // value generator
+      if((((HiveAggregate)rel).getAggCallList().isEmpty() == true
+          && ((HiveAggregate)rel).indicator == false)) {
+        this.mightRequireValueGen = false;
+        return super.visit(rel);
+      }
+      else {
+        // need to reset to true in case previous aggregate/project
+        // has set it to false
+        this.mightRequireValueGen = true;
+        return rel;
+      }
+    }
+    @Override public RelNode visit(LogicalAggregate rel) {
+      if((((LogicalAggregate)rel).getAggCallList().isEmpty() == true
+          && ((LogicalAggregate)rel).indicator == false)) {
+        this.mightRequireValueGen = false;
+        return super.visit(rel);
+      }
+      else {
+        // need to reset to true in case previous aggregate/project
+        // has set it to false
+        this.mightRequireValueGen = true;
+        return rel;
+      }
+    }
+    @Override public RelNode visit(LogicalCorrelate rel) {
+      // this means we are hitting nested subquery so don't
+      // need to go further
+      return rel;
+    }
+
+    public boolean traverse(RelNode root) {
+      root.accept(this);
+      return mightRequireValueGen;
+    }
+  }
   /** Builds a {@link org.apache.calcite.sql2rel.RelDecorrelator.CorelMap}. */
   private static class CorelMapBuilder extends HiveRelShuttleImpl {
     final SortedMap<CorrelationId, RelNode> mapCorToCorRel =

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/queries/clientpositive/subquery_in.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/subquery_in.q b/ql/src/test/queries/clientpositive/subquery_in.q
index 4ba170a..33cc2fe 100644
--- a/ql/src/test/queries/clientpositive/subquery_in.q
+++ b/ql/src/test/queries/clientpositive/subquery_in.q
@@ -74,25 +74,47 @@ from part b where b.p_size in
 ;
 
 -- distinct, corr
-explain 
-select * 
-from src b 
+explain
+select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 ;
 
-select * 
-from src b 
+select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 ;
 
+-- corr, non equi predicate, should not have a join with outer to generate
+-- corr values
+explain
+select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+;
+
+select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+;
+
+
 -- non agg, non corr, windowing
 select p_mfgr, p_name, p_size 
 from part 
@@ -267,3 +289,13 @@ select * from t where i IN (select sum(i) from tt where tt.j = t.j);
 
 drop table t;
 drop table tt;
+
+-- since inner query has aggregate it will be joined with outer to get all possible corrrelated values
+explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name);
+select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name);
+
+-- inner query has join so should have a join with outer query to fetch all corr values
+explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name);
+select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name);
+
+

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
index 8c7f9d3..98b896c 100644
--- a/ql/src/test/results/clientpositive/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
@@ -107,22 +107,22 @@ STAGE PLANS:
             alias: lineitem
             Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: ((l_shipmode = 'AIR') and (l_linenumber = l_linenumber)) (type: boolean)
-              Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+              predicate: ((l_shipmode = 'AIR') and l_linenumber is not null) (type: boolean)
+              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: l_orderkey (type: int), l_linenumber (type: int)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   keys: _col0 (type: int), _col1 (type: int)
                   mode: hash
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     key expressions: _col0 (type: int), _col1 (type: int)
                     sort order: ++
                     Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
-                    Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
index e78ceb3..de8dec1 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
@@ -1743,21 +1743,21 @@ Stage-0
             <-Reducer 3 [SIMPLE_EDGE] llap
               SHUFFLE [RS_15]
                 PartitionCols:_col0
-                Select Operator [SEL_13] (rows=36 width=95)
+                Select Operator [SEL_13] (rows=72 width=95)
                   Output:["_col0","_col1"]
-                  Group By Operator [GBY_12] (rows=36 width=91)
+                  Group By Operator [GBY_12] (rows=72 width=91)
                     Output:["_col0"],keys:_col1
-                    Select Operator [SEL_8] (rows=41 width=178)
+                    Select Operator [SEL_8] (rows=83 width=178)
                       Output:["_col1"]
-                      Group By Operator [GBY_7] (rows=41 width=178)
+                      Group By Operator [GBY_7] (rows=83 width=178)
                         Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
                       <-Map 1 [SIMPLE_EDGE] llap
                         SHUFFLE [RS_6]
                           PartitionCols:_col0
-                          Group By Operator [GBY_5] (rows=41 width=178)
+                          Group By Operator [GBY_5] (rows=83 width=178)
                             Output:["_col0","_col1"],keys:value, key
-                            Filter Operator [FIL_21] (rows=83 width=178)
-                              predicate:((value = value) and (value > 'val_2'))
+                            Filter Operator [FIL_21] (rows=166 width=178)
+                              predicate:(value > 'val_2')
                                Please refer to the previous TableScan [TS_0]
 
 PREHOOK: query: explain select * 
@@ -1814,17 +1814,17 @@ Stage-0
             <-Reducer 4 [SIMPLE_EDGE] llap
               SHUFFLE [RS_14]
                 PartitionCols:_col0, _col1
-                Select Operator [SEL_12] (rows=20 width=182)
+                Select Operator [SEL_12] (rows=83 width=182)
                   Output:["_col0","_col1","_col2"]
-                  Group By Operator [GBY_11] (rows=20 width=178)
+                  Group By Operator [GBY_11] (rows=83 width=178)
                     Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
                   <-Map 1 [SIMPLE_EDGE] llap
                     SHUFFLE [RS_10]
                       PartitionCols:_col0, _col1
-                      Group By Operator [GBY_9] (rows=20 width=178)
+                      Group By Operator [GBY_9] (rows=83 width=178)
                         Output:["_col0","_col1"],keys:key, value
-                        Filter Operator [FIL_20] (rows=41 width=178)
-                          predicate:((value = value) and (key = key) and (value > 'val_12'))
+                        Filter Operator [FIL_20] (rows=166 width=178)
+                          predicate:((value > 'val_12') and key is not null)
                            Please refer to the previous TableScan [TS_0]
 
 PREHOOK: query: create view cv1 as 
@@ -1878,12 +1878,12 @@ Stage-0
         <-Map 3 [SIMPLE_EDGE] llap
           SHUFFLE [RS_8]
             PartitionCols:_col0, _col1
-            Group By Operator [GBY_6] (rows=20 width=178)
+            Group By Operator [GBY_6] (rows=83 width=178)
               Output:["_col0","_col1"],keys:_col0, _col1
-              Select Operator [SEL_4] (rows=41 width=178)
+              Select Operator [SEL_4] (rows=166 width=178)
                 Output:["_col0","_col1"]
-                Filter Operator [FIL_15] (rows=41 width=178)
-                  predicate:((value = value) and (key = key) and (value > 'val_9'))
+                Filter Operator [FIL_15] (rows=166 width=178)
+                  predicate:((value > 'val_9') and key is not null)
                   TableScan [TS_2] (rows=500 width=178)
                     default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
 
@@ -1928,12 +1928,12 @@ Stage-0
         <-Map 3 [SIMPLE_EDGE] llap
           SHUFFLE [RS_8]
             PartitionCols:_col0, _col1
-            Group By Operator [GBY_6] (rows=20 width=178)
+            Group By Operator [GBY_6] (rows=83 width=178)
               Output:["_col0","_col1"],keys:_col0, _col1
-              Select Operator [SEL_4] (rows=41 width=178)
+              Select Operator [SEL_4] (rows=166 width=178)
                 Output:["_col0","_col1"]
-                Filter Operator [FIL_15] (rows=41 width=178)
-                  predicate:((value = value) and (key = key) and (value > 'val_9'))
+                Filter Operator [FIL_15] (rows=166 width=178)
+                  predicate:((value > 'val_9') and key is not null)
                   TableScan [TS_2] (rows=500 width=178)
                     default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
 
@@ -2009,12 +2009,12 @@ Stage-0
           <-Map 6 [SIMPLE_EDGE] llap
             SHUFFLE [RS_19]
               PartitionCols:_col0, _col1
-              Group By Operator [GBY_17] (rows=2 width=8)
+              Group By Operator [GBY_17] (rows=4 width=8)
                 Output:["_col0","_col1"],keys:_col0, _col1
-                Select Operator [SEL_12] (rows=7 width=8)
+                Select Operator [SEL_12] (rows=14 width=8)
                   Output:["_col0","_col1"]
-                  Filter Operator [FIL_30] (rows=7 width=96)
-                    predicate:((l_shipmode = 'AIR') and (l_linenumber = l_linenumber))
+                  Filter Operator [FIL_30] (rows=14 width=96)
+                    predicate:((l_shipmode = 'AIR') and l_linenumber is not null)
                     TableScan [TS_10] (rows=100 width=96)
                       default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"]
           <-Reducer 3 [SIMPLE_EDGE] llap
@@ -2321,36 +2321,36 @@ Stage-0
                 <-Reducer 4 [SIMPLE_EDGE] llap
                   SHUFFLE [RS_19]
                     PartitionCols:_col0
-                    Group By Operator [GBY_7] (rows=1 width=114)
+                    Group By Operator [GBY_7] (rows=2 width=114)
                       Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0
                     <-Map 1 [SIMPLE_EDGE] llap
                       SHUFFLE [RS_6]
                         PartitionCols:_col0
-                        Group By Operator [GBY_5] (rows=1 width=114)
+                        Group By Operator [GBY_5] (rows=2 width=114)
                           Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr
-                          Select Operator [SEL_4] (rows=4 width=223)
+                          Select Operator [SEL_4] (rows=8 width=223)
                             Output:["p_name","p_mfgr"]
-                            Filter Operator [FIL_29] (rows=4 width=223)
-                              predicate:((p_size < 10) and (p_mfgr = p_mfgr))
+                            Filter Operator [FIL_29] (rows=8 width=223)
+                              predicate:((p_size < 10) and p_mfgr is not null)
                                Please refer to the previous TableScan [TS_0]
             <-Reducer 5 [SIMPLE_EDGE] llap
               SHUFFLE [RS_22]
                 PartitionCols:_col0, _col1
-                Select Operator [SEL_17] (rows=2 width=223)
+                Select Operator [SEL_17] (rows=4 width=223)
                   Output:["_col0","_col1","_col2"]
-                  Filter Operator [FIL_16] (rows=2 width=219)
+                  Filter Operator [FIL_16] (rows=4 width=219)
                     predicate:_col0 is not null
-                    Group By Operator [GBY_14] (rows=2 width=219)
+                    Group By Operator [GBY_14] (rows=4 width=219)
                       Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
                     <-Map 1 [SIMPLE_EDGE] llap
                       SHUFFLE [RS_13]
                         PartitionCols:_col0, _col1
-                        Group By Operator [GBY_12] (rows=2 width=219)
+                        Group By Operator [GBY_12] (rows=4 width=219)
                           Output:["_col0","_col1"],keys:p_name, p_mfgr
-                          Select Operator [SEL_11] (rows=4 width=223)
+                          Select Operator [SEL_11] (rows=8 width=223)
                             Output:["p_name","p_mfgr"]
-                            Filter Operator [FIL_30] (rows=4 width=223)
-                              predicate:((p_size < 10) and (p_mfgr = p_mfgr))
+                            Filter Operator [FIL_30] (rows=8 width=223)
+                              predicate:((p_size < 10) and p_mfgr is not null)
                                Please refer to the previous TableScan [TS_0]
 
 PREHOOK: query: explain select p_name, p_size 
@@ -2509,7 +2509,7 @@ Stage-0
                               Select Operator [SEL_12] (rows=1 width=114)
                                 Output:["_col0","_col1"]
                                 Filter Operator [FIL_40] (rows=1 width=114)
-                                  predicate:((_col1 = _col1) and ((_col2 - _col1) > 600.0))
+                                  predicate:(((_col2 - _col1) > 600.0) and _col1 is not null)
                                   Group By Operator [GBY_10] (rows=5 width=114)
                                     Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0
                                   <-Map 1 [SIMPLE_EDGE] llap
@@ -2528,7 +2528,7 @@ Stage-0
                         Select Operator [SEL_24] (rows=1 width=110)
                           Output:["_col0","_col1"]
                           Filter Operator [FIL_41] (rows=1 width=114)
-                            predicate:((_col1 = _col1) and ((_col2 - _col1) > 600.0))
+                            predicate:(((_col2 - _col1) > 600.0) and _col1 is not null)
                             Group By Operator [GBY_22] (rows=5 width=114)
                               Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0
                             <-Map 1 [SIMPLE_EDGE] llap

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_exists.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_exists.q.out b/ql/src/test/results/clientpositive/llap/subquery_exists.q.out
index 3004e36..0749872 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_exists.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_exists.q.out
@@ -50,22 +50,22 @@ STAGE PLANS:
                   alias: a
                   Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key = key) and (value > 'val_9')) (type: boolean)
-                    Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((value > 'val_9') and key is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 

http://git-wip-us.apache.org/repos/asf/hive/blob/3192d06d/ql/src/test/results/clientpositive/llap/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out
index b3d735a..e401f31 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out
@@ -164,22 +164,22 @@ STAGE PLANS:
                   alias: a
                   Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key > '9')) (type: boolean)
-                    Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((key > '9') and value is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
@@ -463,13 +463,13 @@ STAGE PLANS:
                   alias: part
                   Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: (p_mfgr = p_mfgr) (type: boolean)
-                    Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: p_mfgr is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                     Reduce Output Operator
                       key expressions: p_mfgr (type: string), p_size (type: int)
                       sort order: ++
                       Map-reduce partition columns: p_mfgr (type: string)
-                      Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
                       TopN Hash Memory Usage: 0.1
             Execution mode: llap
             LLAP IO: no inputs
@@ -501,7 +501,7 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
                 outputColumnNames: _col2, _col5
-                Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                 PTF Operator
                   Function definitions:
                       Input definition
@@ -522,25 +522,25 @@ STAGE PLANS:
                               window function: GenericUDAFRankEvaluator
                               window frame: PRECEDING(MAX)~FOLLOWING(MAX)
                               isPivotResult: true
-                  Statistics: Num rows: 13 Data size: 4810 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
                     predicate: (rank_window_0 <= 2) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: _col2 (type: string), _col5 (type: int)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 1480 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         aggregations: min(_col1)
                         keys: _col0 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                           value expressions: _col1 (type: int)
         Reducer 5 
             Execution mode: llap
@@ -550,16 +550,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: string)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col1 (type: int), _col0 (type: string)
                   outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col1 (type: string), _col0 (type: int)
                     sort order: ++
                     Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
-                    Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
 
   Stage: Stage-0
     Fetch Operator
@@ -591,21 +591,21 @@ Manufacturer#2	almond aquamarine midnight light salmon	2
 Manufacturer#3	almond antique misty red olive	1
 Manufacturer#4	almond aquamarine yellow dodger mint	7
 Manufacturer#5	almond antique sky peru orange	2
-PREHOOK: query: explain 
-select * 
-from src b 
+PREHOOK: query: explain
+select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 PREHOOK: type: QUERY
-POSTHOOK: query: explain 
-select * 
-from src b 
+POSTHOOK: query: explain
+select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 POSTHOOK: type: QUERY
@@ -643,22 +643,22 @@ STAGE PLANS:
                   alias: a
                   Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((value = value) and (key > '9')) (type: boolean)
-                    Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((key > '9') and value is not null) (type: boolean)
+                    Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 41 Data size: 7298 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
@@ -686,21 +686,21 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-PREHOOK: query: select * 
-from src b 
+PREHOOK: query: select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 #### A masked pattern was here ####
-POSTHOOK: query: select * 
-from src b 
+POSTHOOK: query: select *
+from src b
 where b.key in
-        (select distinct a.key 
-         from src a 
+        (select distinct a.key
+         from src a
          where b.value = a.value and a.key > '9'
         )
 POSTHOOK: type: QUERY
@@ -717,6 +717,148 @@ POSTHOOK: Input: default@src
 97	val_97
 98	val_98
 98	val_98
+PREHOOK: query: explain
+select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+        Reducer 4 <- Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+                  Select Operator
+                    expressions: key (type: string), value (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: string)
+                      Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+                      value expressions: _col1 (type: string)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: a
+                  Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: (key > '9') (type: boolean)
+                    Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
+                    Group By Operator
+                      keys: key (type: string)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Merge Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col0 (type: string)
+                  1 _col0 (type: string)
+                outputColumnNames: _col0, _col1, _col3
+                residual filter predicates: {(_col1 <> _col3)}
+                Statistics: Num rows: 168 Data size: 44520 Basic stats: COMPLETE Column stats: COMPLETE
+                Select Operator
+                  expressions: _col0 (type: string), _col1 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 168 Data size: 29904 Basic stats: COMPLETE Column stats: COMPLETE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 168 Data size: 29904 Basic stats: COMPLETE Column stats: COMPLETE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 4 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE
+                Reduce Output Operator
+                  key expressions: _col0 (type: string)
+                  sort order: +
+                  Map-reduce partition columns: _col0 (type: string)
+                  Statistics: Num rows: 69 Data size: 6003 Basic stats: COMPLETE Column stats: COMPLETE
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select *
+from src b
+where b.key in
+        (select distinct a.key
+         from src a
+         where b.value <> a.key and a.key > '9'
+        )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+90	val_90
+90	val_90
+90	val_90
+92	val_92
+95	val_95
+95	val_95
+96	val_96
+97	val_97
+97	val_97
+98	val_98
+98	val_98
 PREHOOK: query: select p_mfgr, p_name, p_size 
 from part 
 where part.p_size in 
@@ -1967,18 +2109,18 @@ STAGE PLANS:
                   alias: p
                   Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((p_size = p_size) and (p_partkey = p_partkey)) (type: boolean)
-                    Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: (p_size is not null and p_partkey is not null) (type: boolean)
+                    Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
                     Group By Operator
                       keys: p_partkey (type: int), p_name (type: string), p_size (type: int)
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2
-                      Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int)
-                        Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
@@ -2006,16 +2148,16 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                 Select Operator
                   expressions: _col1 (type: string), _col0 (type: int), _col2 (type: int)
                   outputColumnNames: _col0, _col1, _col2
-                  Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col1 (type: int), _col0 (type: string), _col2 (type: int)
                     sort order: +++
                     Map-reduce partition columns: _col1 (type: int), _col0 (type: string), _col2 (type: int)
-                    Statistics: Num rows: 3 Data size: 387 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 13 Data size: 1677 Basic stats: COMPLETE Column stats: COMPLETE
 
   Stage: Stage-0
     Fetch Operator
@@ -3275,22 +3417,22 @@ STAGE PLANS:
                   alias: part
                   Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: ((p_size < 10) and (p_mfgr = p_mfgr)) (type: boolean)
-                    Statistics: Num rows: 4 Data size: 892 Basic stats: COMPLETE Column stats: COMPLETE
+                    predicate: ((p_size < 10) and p_mfgr is not null) (type: boolean)
+                    Statistics: Num rows: 8 Data size: 1784 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: p_mfgr (type: string), p_name (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 8 Data size: 1752 Basic stats: COMPLETE Column stats: COMPLETE
                       Group By Operator
                         keys: _col0 (type: string), _col1 (type: string)
                         mode: hash
                         outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
                         Reduce Output Operator
                           key expressions: _col0 (type: string), _col1 (type: string)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 2 Data size: 438 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 4 Data size: 876 Basic stats: COMPLETE Column stats: COMPLETE
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
@@ -5273,3 +5415,386 @@ POSTHOOK: query: drop table tt
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@tt
 POSTHOOK: Output: default@tt
+Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product
+PREHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+        Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE)
+        Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+        Reducer 7 <- Map 6 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE
+                  Select Operator
+                    expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string)
+                    outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                    Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col1 (type: string), _col5 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col1 (type: string), _col5 (type: int)
+                      Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE
+                      value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: p
+                  Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                  Select Operator
+                    expressions: p_type (type: string), p_size (type: int)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      sort order: 
+                      Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                      value expressions: _col0 (type: string), _col1 (type: int)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Map 6 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE
+                  Group By Operator
+                    keys: p_name (type: string)
+                    mode: hash
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: string)
+                      Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Merge Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col1 (type: string), _col5 (type: int)
+                  1 _col1 (type: string), _col0 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                Statistics: Num rows: 1 Data size: 619 Basic stats: COMPLETE Column stats: COMPLETE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 619 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 4 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Merge Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2
+                residual filter predicates: {(_col0 <> _col2)}
+                Statistics: Num rows: 338 Data size: 77402 Basic stats: COMPLETE Column stats: COMPLETE
+                Select Operator
+                  expressions: _col1 (type: int), _col2 (type: string)
+                  outputColumnNames: _col1, _col2
+                  Statistics: Num rows: 338 Data size: 77402 Basic stats: COMPLETE Column stats: COMPLETE
+                  Group By Operator
+                    aggregations: max(_col1)
+                    keys: _col2 (type: string)
+                    mode: hash
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: string)
+                      Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE
+                      value expressions: _col1 (type: int)
+        Reducer 5 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: max(VALUE._col0)
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE
+                Select Operator
+                  expressions: _col1 (type: int), _col0 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE
+                  Reduce Output Operator
+                    key expressions: _col1 (type: string), _col0 (type: int)
+                    sort order: ++
+                    Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
+                    Statistics: Num rows: 25 Data size: 3125 Basic stats: COMPLETE Column stats: COMPLETE
+        Reducer 7 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: COMPLETE
+                  value expressions: _col0 (type: string)
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+Warning: Shuffle Join MERGEJOIN[29][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product
+PREHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part where p_size IN (select max(p_size) from part p where p.p_type <> part.p_name)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+15103	almond aquamarine dodger light gainsboro	Manufacturer#5	Brand#53	ECONOMY BURNISHED STEEL	46	LG PACK	1018.1	packages hinder carefu
+Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product
+PREHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+        Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
+        Reducer 5 <- Map 9 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+        Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+        Reducer 8 <- Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE
+                  Select Operator
+                    expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string)
+                    outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                    Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col4 (type: string), _col5 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col4 (type: string), _col5 (type: int)
+                      Statistics: Num rows: 26 Data size: 16094 Basic stats: COMPLETE Column stats: COMPLETE
+                      value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col6 (type: string), _col7 (type: double), _col8 (type: string)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: p
+                  Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: p_type is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: p_name (type: string), p_type (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 26 Data size: 5850 Basic stats: COMPLETE Column stats: COMPLETE
+                        value expressions: _col0 (type: string), _col1 (type: string)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Map 7 
+            Map Operator Tree:
+                TableScan
+                  alias: part
+                  Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE
+                  Group By Operator
+                    keys: p_type (type: string)
+                    mode: hash
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: string)
+                      Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE
+            Execution mode: llap
+            LLAP IO: no inputs
+        Map 9 
+            Map Operator Tree:
+                TableScan
+                  alias: pp
+                  Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: p_type is not null (type: boolean)
+                    Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: p_type (type: string), p_size (type: int)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
+                        value expressions: _col1 (type: int)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Merge Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col4 (type: string), _col5 (type: int)
+                  1 _col1 (type: string), _col0 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 8 Data size: 4952 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 4 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Merge Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 
+                  1 
+                outputColumnNames: _col0, _col1, _col2
+                residual filter predicates: {(_col2 <> _col0)}
+                Statistics: Num rows: 338 Data size: 111202 Basic stats: COMPLETE Column stats: COMPLETE
+                Reduce Output Operator
+                  key expressions: _col1 (type: string)
+                  sort order: +
+                  Map-reduce partition columns: _col1 (type: string)
+                  Statistics: Num rows: 338 Data size: 111202 Basic stats: COMPLETE Column stats: COMPLETE
+                  value expressions: _col2 (type: string)
+        Reducer 5 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Merge Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col1 (type: string)
+                  1 _col0 (type: string)
+                outputColumnNames: _col2, _col4
+                Statistics: Num rows: 313 Data size: 33804 Basic stats: COMPLETE Column stats: COMPLETE
+                Group By Operator
+                  keys: _col2 (type: string), _col4 (type: int)
+                  mode: hash
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 156 Data size: 16848 Basic stats: COMPLETE Column stats: COMPLETE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string), _col1 (type: int)
+                    sort order: ++
+                    Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+                    Statistics: Num rows: 156 Data size: 16848 Basic stats: COMPLETE Column stats: COMPLETE
+        Reducer 6 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string), KEY._col1 (type: int)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 156 Data size: 16848 Basic stats: COMPLETE Column stats: COMPLETE
+                Select Operator
+                  expressions: _col1 (type: int), _col0 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 156 Data size: 16848 Basic stats: COMPLETE Column stats: COMPLETE
+                  Reduce Output Operator
+                    key expressions: _col1 (type: string), _col0 (type: int)
+                    sort order: ++
+                    Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
+                    Statistics: Num rows: 156 Data size: 16848 Basic stats: COMPLETE Column stats: COMPLETE
+        Reducer 8 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: string)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE
+                  value expressions: _col0 (type: string)
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+Warning: Shuffle Join MERGEJOIN[41][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product
+PREHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part where p_size IN (select pp.p_size from part p join part pp on pp.p_type = p.p_type where part.p_type <> p.p_name)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+105685	almond antique violet chocolate turquoise	Manufacturer#2	Brand#22	MEDIUM ANODIZED COPPER	14	MED CAN	1690.68	ly pending requ
+110592	almond antique salmon chartreuse burlywood	Manufacturer#1	Brand#15	PROMO BURNISHED NICKEL	6	JUMBO PKG	1602.59	 to the furiously
+112398	almond antique metallic orange dim	Manufacturer#3	Brand#32	MEDIUM BURNISHED BRASS	19	JUMBO JAR	1410.39	ole car
+121152	almond antique burnished rose metallic	Manufacturer#1	Brand#14	PROMO PLATED TIN	2	JUMBO BOX	1173.15	e pinto beans h
+121152	almond antique burnished rose metallic	Manufacturer#1	Brand#14	PROMO PLATED TIN	2	JUMBO BOX	1173.15	e pinto beans h
+132666	almond aquamarine rose maroon antique	Manufacturer#2	Brand#24	SMALL POLISHED NICKEL	25	MED BOX	1698.66	even 
+144293	almond antique olive coral navajo	Manufacturer#3	Brand#34	STANDARD POLISHED STEEL	45	JUMBO CAN	1337.29	ag furiously about 
+146985	almond aquamarine midnight light salmon	Manufacturer#2	Brand#23	MEDIUM BURNISHED COPPER	2	SM CASE	2031.98	s cajole caref
+15103	almond aquamarine dodger light gainsboro	Manufacturer#5	Brand#53	ECONOMY BURNISHED STEEL	46	LG PACK	1018.1	packages hinder carefu
+155733	almond antique sky peru orange	Manufacturer#5	Brand#53	SMALL PLATED BRASS	2	WRAP DRUM	1788.73	furiously. bra
+17273	almond antique forest lavender goldenrod	Manufacturer#3	Brand#35	PROMO ANODIZED TIN	14	JUMBO CASE	1190.27	along the
+17927	almond aquamarine yellow dodger mint	Manufacturer#4	Brand#41	ECONOMY BRUSHED COPPER	7	SM PKG	1844.92	ites. eve
+191709	almond antique violet turquoise frosted	Manufacturer#2	Brand#22	ECONOMY POLISHED STEEL	40	MED BOX	1800.7	 haggle
+192697	almond antique blue firebrick mint	Manufacturer#5	Brand#52	MEDIUM BURNISHED TIN	31	LG DRUM	1789.69	ickly ir
+195606	almond aquamarine sandy cyan gainsboro	Manufacturer#2	Brand#25	STANDARD PLATED TIN	18	SM PKG	1701.6	ic de
+33357	almond azure aquamarine papaya violet	Manufacturer#4	Brand#41	STANDARD ANODIZED TIN	12	WRAP CASE	1290.35	reful
+40982	almond antique misty red olive	Manufacturer#3	Brand#32	ECONOMY PLATED COPPER	1	LG PKG	1922.98	c foxes can s
+42669	almond antique medium spring khaki	Manufacturer#5	Brand#51	STANDARD BURNISHED TIN	6	MED CAN	1611.66	sits haggl
+45261	almond aquamarine floral ivory bisque	Manufacturer#4	Brand#42	SMALL PLATED STEEL	27	WRAP CASE	1206.26	careful
+48427	almond antique violet mint lemon	Manufacturer#4	Brand#42	PROMO POLISHED STEEL	39	SM CASE	1375.42	hely ironic i
+49671	almond antique gainsboro frosted violet	Manufacturer#4	Brand#41	SMALL BRUSHED BRASS	10	SM BOX	1620.67	ccounts run quick
+65667	almond aquamarine pink moccasin thistle	Manufacturer#1	Brand#12	LARGE BURNISHED STEEL	42	JUMBO CASE	1632.66	e across the expr
+78486	almond azure blanched chiffon midnight	Manufacturer#5	Brand#52	LARGE BRUSHED BRASS	23	MED BAG	1464.48	hely blith
+85768	almond antique chartreuse lavender yellow	Manufacturer#1	Brand#12	LARGE BRUSHED STEEL	34	SM BAG	1753.76	refull
+86428	almond aquamarine burnished black steel	Manufacturer#1	Brand#12	STANDARD ANODIZED STEEL	28	WRAP BAG	1414.42	arefully 
+90681	almond antique chartreuse khaki white	Manufacturer#3	Brand#31	MEDIUM BURNISHED TIN	17	SM CASE	1671.68	are slyly after the sl