You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/10/21 19:47:07 UTC

[1/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Repository: hive
Updated Branches:
  refs/heads/master 0d39a81c1 -> 5eebbdf7c


http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
index 5ae15fe..2ab11da 100644
--- a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
@@ -57,139 +57,139 @@ POSTHOOK: Input: default@flights_tiny_orc
 Baltimore	New York	2010-10-20	2010-10-20 07:00:00	-30.0	1064
 Baltimore	New York	2010-10-20	2010-10-20 07:00:00	23.0	1142
 Baltimore	New York	2010-10-20	2010-10-20 07:00:00	6.0	1599
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	42.0	361
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	24.0	897
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	15.0	1531
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	-6.0	1610
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	-2.0	3198
-Baltimore	New York	2010-10-21	2010-10-21 07:00:00	17.0	1064
 Baltimore	New York	2010-10-21	2010-10-21 07:00:00	105.0	1142
+Baltimore	New York	2010-10-21	2010-10-21 07:00:00	17.0	1064
 Baltimore	New York	2010-10-21	2010-10-21 07:00:00	28.0	1599
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	142.0	361
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	77.0	897
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	53.0	1531
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	-5.0	1610
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	51.0	3198
 Baltimore	New York	2010-10-22	2010-10-22 07:00:00	-12.0	1064
-Baltimore	New York	2010-10-22	2010-10-22 07:00:00	54.0	1142
 Baltimore	New York	2010-10-22	2010-10-22 07:00:00	18.0	1599
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	2.0	361
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	24.0	897
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	16.0	1531
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	-6.0	1610
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	-11.0	3198
-Baltimore	New York	2010-10-23	2010-10-23 07:00:00	18.0	272
+Baltimore	New York	2010-10-22	2010-10-22 07:00:00	54.0	1142
 Baltimore	New York	2010-10-23	2010-10-23 07:00:00	-10.0	1805
+Baltimore	New York	2010-10-23	2010-10-23 07:00:00	18.0	272
 Baltimore	New York	2010-10-23	2010-10-23 07:00:00	6.0	3171
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	3.0	384
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	32.0	426
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	1.0	650
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	11.0	3085
 Baltimore	New York	2010-10-24	2010-10-24 07:00:00	12.0	1599
 Baltimore	New York	2010-10-24	2010-10-24 07:00:00	20.0	2571
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	10.0	361
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	113.0	897
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	-5.0	1531
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	-17.0	1610
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	-3.0	3198
 Baltimore	New York	2010-10-25	2010-10-25 07:00:00	-25.0	1064
-Baltimore	New York	2010-10-25	2010-10-25 07:00:00	92.0	1142
 Baltimore	New York	2010-10-25	2010-10-25 07:00:00	106.0	1599
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	31.0	361
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	-1.0	897
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	43.0	1531
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	6.0	1610
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	-16.0	3198
+Baltimore	New York	2010-10-25	2010-10-25 07:00:00	92.0	1142
 Baltimore	New York	2010-10-26	2010-10-26 07:00:00	-22.0	1064
 Baltimore	New York	2010-10-26	2010-10-26 07:00:00	123.0	1142
 Baltimore	New York	2010-10-26	2010-10-26 07:00:00	90.0	1599
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	12.0	361
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	0.0	897
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	29.0	1531
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	-17.0	1610
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	6.0	3198
 Baltimore	New York	2010-10-27	2010-10-27 07:00:00	-18.0	1064
 Baltimore	New York	2010-10-27	2010-10-27 07:00:00	49.0	1142
 Baltimore	New York	2010-10-27	2010-10-27 07:00:00	92.0	1599
-Chicago	New York	2010-10-27	2010-10-27 07:00:00	148.0	361
+Baltimore	New York	2010-10-28	2010-10-28 07:00:00	-14.0	1142
+Baltimore	New York	2010-10-28	2010-10-28 07:00:00	-14.0	1599
+Baltimore	New York	2010-10-28	2010-10-28 07:00:00	-4.0	1064
+Baltimore	New York	2010-10-29	2010-10-29 07:00:00	-2.0	1599
+Baltimore	New York	2010-10-29	2010-10-29 07:00:00	-24.0	1064
+Baltimore	New York	2010-10-29	2010-10-29 07:00:00	21.0	1142
+Baltimore	New York	2010-10-30	2010-10-30 07:00:00	-1.0	1805
+Baltimore	New York	2010-10-30	2010-10-30 07:00:00	14.0	272
+Baltimore	New York	2010-10-30	2010-10-30 07:00:00	5.0	3171
+Baltimore	New York	2010-10-31	2010-10-31 07:00:00	-1.0	1599
+Baltimore	New York	2010-10-31	2010-10-31 07:00:00	-14.0	2571
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	-2.0	3198
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	-6.0	1610
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	15.0	1531
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	24.0	897
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	42.0	361
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	-5.0	1610
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	142.0	361
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	51.0	3198
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	53.0	1531
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	77.0	897
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	-11.0	3198
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	-6.0	1610
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	16.0	1531
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	2.0	361
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	24.0	897
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	1.0	650
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	11.0	3085
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	3.0	384
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	32.0	426
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	-17.0	1610
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	-3.0	3198
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	-5.0	1531
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	10.0	361
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	113.0	897
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	-1.0	897
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	-16.0	3198
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	31.0	361
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	43.0	1531
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	6.0	1610
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	-17.0	1610
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	0.0	897
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	12.0	361
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	29.0	1531
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	6.0	3198
 Chicago	New York	2010-10-27	2010-10-27 07:00:00	-11.0	897
+Chicago	New York	2010-10-27	2010-10-27 07:00:00	148.0	361
+Chicago	New York	2010-10-27	2010-10-27 07:00:00	21.0	3198
 Chicago	New York	2010-10-27	2010-10-27 07:00:00	70.0	1531
 Chicago	New York	2010-10-27	2010-10-27 07:00:00	8.0	1610
-Chicago	New York	2010-10-27	2010-10-27 07:00:00	21.0	3198
-Baltimore	New York	2010-10-28	2010-10-28 07:00:00	-4.0	1064
-Baltimore	New York	2010-10-28	2010-10-28 07:00:00	-14.0	1142
-Baltimore	New York	2010-10-28	2010-10-28 07:00:00	-14.0	1599
+Chicago	New York	2010-10-28	2010-10-28 07:00:00	-11.0	1531
+Chicago	New York	2010-10-28	2010-10-28 07:00:00	-18.0	3198
 Chicago	New York	2010-10-28	2010-10-28 07:00:00	2.0	361
 Chicago	New York	2010-10-28	2010-10-28 07:00:00	2.0	897
-Chicago	New York	2010-10-28	2010-10-28 07:00:00	-11.0	1531
 Chicago	New York	2010-10-28	2010-10-28 07:00:00	3.0	1610
-Chicago	New York	2010-10-28	2010-10-28 07:00:00	-18.0	3198
-Baltimore	New York	2010-10-29	2010-10-29 07:00:00	-24.0	1064
-Baltimore	New York	2010-10-29	2010-10-29 07:00:00	21.0	1142
-Baltimore	New York	2010-10-29	2010-10-29 07:00:00	-2.0	1599
-Chicago	New York	2010-10-29	2010-10-29 07:00:00	-12.0	361
 Chicago	New York	2010-10-29	2010-10-29 07:00:00	-11.0	897
-Chicago	New York	2010-10-29	2010-10-29 07:00:00	15.0	1531
+Chicago	New York	2010-10-29	2010-10-29 07:00:00	-12.0	361
 Chicago	New York	2010-10-29	2010-10-29 07:00:00	-18.0	1610
 Chicago	New York	2010-10-29	2010-10-29 07:00:00	-4.0	3198
-Baltimore	New York	2010-10-30	2010-10-30 07:00:00	14.0	272
-Baltimore	New York	2010-10-30	2010-10-30 07:00:00	-1.0	1805
-Baltimore	New York	2010-10-30	2010-10-30 07:00:00	5.0	3171
-Chicago	New York	2010-10-30	2010-10-30 07:00:00	-6.0	384
+Chicago	New York	2010-10-29	2010-10-29 07:00:00	15.0	1531
 Chicago	New York	2010-10-30	2010-10-30 07:00:00	-10.0	426
-Chicago	New York	2010-10-30	2010-10-30 07:00:00	-5.0	650
 Chicago	New York	2010-10-30	2010-10-30 07:00:00	-5.0	3085
-Baltimore	New York	2010-10-31	2010-10-31 07:00:00	-1.0	1599
-Baltimore	New York	2010-10-31	2010-10-31 07:00:00	-14.0	2571
-Chicago	New York	2010-10-31	2010-10-31 07:00:00	-25.0	361
+Chicago	New York	2010-10-30	2010-10-30 07:00:00	-5.0	650
+Chicago	New York	2010-10-30	2010-10-30 07:00:00	-6.0	384
+Chicago	New York	2010-10-31	2010-10-31 07:00:00	-15.0	3198
 Chicago	New York	2010-10-31	2010-10-31 07:00:00	-18.0	897
-Chicago	New York	2010-10-31	2010-10-31 07:00:00	-4.0	1531
 Chicago	New York	2010-10-31	2010-10-31 07:00:00	-22.0	1610
-Chicago	New York	2010-10-31	2010-10-31 07:00:00	-15.0	3198
-Cleveland	New York	2010-10-30	2010-10-30 07:00:00	-23.0	2018
-Cleveland	New York	2010-10-30	2010-10-30 07:00:00	-12.0	2932
-Cleveland	New York	2010-10-29	2010-10-29 07:00:00	-4.0	2630
-Cleveland	New York	2010-10-29	2010-10-29 07:00:00	-19.0	2646
-Cleveland	New York	2010-10-29	2010-10-29 07:00:00	-12.0	3014
-Cleveland	New York	2010-10-28	2010-10-28 07:00:00	3.0	2630
-Cleveland	New York	2010-10-28	2010-10-28 07:00:00	-6.0	2646
-Cleveland	New York	2010-10-28	2010-10-28 07:00:00	1.0	3014
-Cleveland	New York	2010-10-27	2010-10-27 07:00:00	16.0	2630
-Cleveland	New York	2010-10-27	2010-10-27 07:00:00	27.0	3014
-Cleveland	New York	2010-10-26	2010-10-26 07:00:00	4.0	2630
-Cleveland	New York	2010-10-26	2010-10-26 07:00:00	-27.0	2646
-Cleveland	New York	2010-10-26	2010-10-26 07:00:00	-11.0	2662
-Cleveland	New York	2010-10-26	2010-10-26 07:00:00	13.0	3014
-Cleveland	New York	2010-10-25	2010-10-25 07:00:00	-4.0	2630
-Cleveland	New York	2010-10-25	2010-10-25 07:00:00	81.0	2646
-Cleveland	New York	2010-10-25	2010-10-25 07:00:00	42.0	3014
-Cleveland	New York	2010-10-24	2010-10-24 07:00:00	5.0	2254
+Chicago	New York	2010-10-31	2010-10-31 07:00:00	-25.0	361
+Chicago	New York	2010-10-31	2010-10-31 07:00:00	-4.0	1531
+Cleveland	New York	2010-10-20	2010-10-20 07:00:00	-15.0	3014
+Cleveland	New York	2010-10-20	2010-10-20 07:00:00	-8.0	2630
+Cleveland	New York	2010-10-21	2010-10-21 07:00:00	29.0	2646
+Cleveland	New York	2010-10-21	2010-10-21 07:00:00	3.0	2630
+Cleveland	New York	2010-10-21	2010-10-21 07:00:00	72.0	3014
+Cleveland	New York	2010-10-22	2010-10-22 07:00:00	-25.0	2646
+Cleveland	New York	2010-10-22	2010-10-22 07:00:00	-3.0	3014
+Cleveland	New York	2010-10-22	2010-10-22 07:00:00	1.0	2630
+Cleveland	New York	2010-10-23	2010-10-23 07:00:00	-21.0	2932
 Cleveland	New York	2010-10-24	2010-10-24 07:00:00	-11.0	2630
 Cleveland	New York	2010-10-24	2010-10-24 07:00:00	-20.0	2646
 Cleveland	New York	2010-10-24	2010-10-24 07:00:00	-9.0	3014
-Cleveland	New York	2010-10-23	2010-10-23 07:00:00	-21.0	2932
-Cleveland	New York	2010-10-22	2010-10-22 07:00:00	1.0	2630
-Cleveland	New York	2010-10-22	2010-10-22 07:00:00	-25.0	2646
-Cleveland	New York	2010-10-22	2010-10-22 07:00:00	-3.0	3014
-Cleveland	New York	2010-10-21	2010-10-21 07:00:00	3.0	2630
-Cleveland	New York	2010-10-21	2010-10-21 07:00:00	29.0	2646
-Cleveland	New York	2010-10-21	2010-10-21 07:00:00	72.0	3014
-Cleveland	New York	2010-10-20	2010-10-20 07:00:00	-8.0	2630
-Cleveland	New York	2010-10-20	2010-10-20 07:00:00	-15.0	3014
-Washington	New York	2010-10-23	2010-10-23 07:00:00	-25.0	5832
-Washington	New York	2010-10-23	2010-10-23 07:00:00	-21.0	5904
-Washington	New York	2010-10-23	2010-10-23 07:00:00	-18.0	5917
-Washington	New York	2010-10-30	2010-10-30 07:00:00	-27.0	5904
-Washington	New York	2010-10-30	2010-10-30 07:00:00	-16.0	5917
+Cleveland	New York	2010-10-24	2010-10-24 07:00:00	5.0	2254
+Cleveland	New York	2010-10-25	2010-10-25 07:00:00	-4.0	2630
+Cleveland	New York	2010-10-25	2010-10-25 07:00:00	42.0	3014
+Cleveland	New York	2010-10-25	2010-10-25 07:00:00	81.0	2646
+Cleveland	New York	2010-10-26	2010-10-26 07:00:00	-11.0	2662
+Cleveland	New York	2010-10-26	2010-10-26 07:00:00	-27.0	2646
+Cleveland	New York	2010-10-26	2010-10-26 07:00:00	13.0	3014
+Cleveland	New York	2010-10-26	2010-10-26 07:00:00	4.0	2630
+Cleveland	New York	2010-10-27	2010-10-27 07:00:00	16.0	2630
+Cleveland	New York	2010-10-27	2010-10-27 07:00:00	27.0	3014
+Cleveland	New York	2010-10-28	2010-10-28 07:00:00	-6.0	2646
+Cleveland	New York	2010-10-28	2010-10-28 07:00:00	1.0	3014
+Cleveland	New York	2010-10-28	2010-10-28 07:00:00	3.0	2630
+Cleveland	New York	2010-10-29	2010-10-29 07:00:00	-12.0	3014
+Cleveland	New York	2010-10-29	2010-10-29 07:00:00	-19.0	2646
+Cleveland	New York	2010-10-29	2010-10-29 07:00:00	-4.0	2630
+Cleveland	New York	2010-10-30	2010-10-30 07:00:00	-12.0	2932
+Cleveland	New York	2010-10-30	2010-10-30 07:00:00	-23.0	2018
 Washington	New York	2010-10-20	2010-10-20 07:00:00	-2.0	7291
 Washington	New York	2010-10-21	2010-10-21 07:00:00	22.0	7291
 Washington	New York	2010-10-23	2010-10-23 07:00:00	-16.0	7274
+Washington	New York	2010-10-23	2010-10-23 07:00:00	-18.0	5917
+Washington	New York	2010-10-23	2010-10-23 07:00:00	-21.0	5904
+Washington	New York	2010-10-23	2010-10-23 07:00:00	-25.0	5832
 Washington	New York	2010-10-24	2010-10-24 07:00:00	-26.0	7282
 Washington	New York	2010-10-25	2010-10-25 07:00:00	9.0	7291
 Washington	New York	2010-10-26	2010-10-26 07:00:00	4.0	7291
 Washington	New York	2010-10-27	2010-10-27 07:00:00	26.0	7291
 Washington	New York	2010-10-28	2010-10-28 07:00:00	45.0	7291
 Washington	New York	2010-10-29	2010-10-29 07:00:00	1.0	7291
+Washington	New York	2010-10-30	2010-10-30 07:00:00	-16.0	5917
+Washington	New York	2010-10-30	2010-10-30 07:00:00	-27.0	5904
 Washington	New York	2010-10-31	2010-10-31 07:00:00	-18.0	7282
 PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25
 PREHOOK: type: QUERY
@@ -201,29 +201,29 @@ POSTHOOK: Input: default@flights_tiny_orc
 #### A masked pattern was here ####
 Baltimore	New York	2010-10-23	2010-10-23 07:00:00	18.0	272
 Baltimore	New York	2010-10-30	2010-10-30 07:00:00	14.0	272
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	24.0	897
 Chicago	New York	2010-10-20	2010-10-20 07:00:00	42.0	361
 Chicago	New York	2010-10-21	2010-10-21 07:00:00	142.0	361
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	77.0	897
 Chicago	New York	2010-10-22	2010-10-22 07:00:00	2.0	361
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	24.0	897
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	1.0	650
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	3.0	384
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	32.0	426
 Chicago	New York	2010-10-24	2010-10-24 07:00:00	10.0	361
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	113.0	897
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	-1.0	897
 Chicago	New York	2010-10-25	2010-10-25 07:00:00	31.0	361
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	0.0	897
 Chicago	New York	2010-10-26	2010-10-26 07:00:00	12.0	361
+Chicago	New York	2010-10-27	2010-10-27 07:00:00	-11.0	897
 Chicago	New York	2010-10-27	2010-10-27 07:00:00	148.0	361
 Chicago	New York	2010-10-28	2010-10-28 07:00:00	2.0	361
 Chicago	New York	2010-10-29	2010-10-29 07:00:00	-12.0	361
-Chicago	New York	2010-10-31	2010-10-31 07:00:00	-25.0	361
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	3.0	384
-Chicago	New York	2010-10-30	2010-10-30 07:00:00	-6.0	384
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	32.0	426
 Chicago	New York	2010-10-30	2010-10-30 07:00:00	-10.0	426
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	1.0	650
 Chicago	New York	2010-10-30	2010-10-30 07:00:00	-5.0	650
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	24.0	897
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	77.0	897
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	24.0	897
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	113.0	897
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	-1.0	897
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	0.0	897
-Chicago	New York	2010-10-27	2010-10-27 07:00:00	-11.0	897
+Chicago	New York	2010-10-30	2010-10-30 07:00:00	-6.0	384
+Chicago	New York	2010-10-31	2010-10-31 07:00:00	-25.0	361
 PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date
 PREHOOK: type: QUERY
 PREHOOK: Input: default@flights_tiny_orc
@@ -232,16 +232,16 @@ POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@flights_tiny_orc
 #### A masked pattern was here ####
+2010-10-20	11
 2010-10-21	12
 2010-10-22	11
-2010-10-29	12
-2010-10-20	11
 2010-10-23	12
 2010-10-24	12
 2010-10-25	12
 2010-10-26	13
 2010-10-27	11
 2010-10-28	12
+2010-10-29	12
 2010-10-30	11
 2010-10-31	8
 PREHOOK: query: explain vectorization expression
@@ -390,29 +390,29 @@ POSTHOOK: Input: default@flights_tiny_orc
 #### A masked pattern was here ####
 Baltimore	New York	2010-10-23	2010-10-23 07:00:00	18.0	272
 Baltimore	New York	2010-10-30	2010-10-30 07:00:00	14.0	272
+Chicago	New York	2010-10-20	2010-10-20 07:00:00	24.0	897
 Chicago	New York	2010-10-20	2010-10-20 07:00:00	42.0	361
 Chicago	New York	2010-10-21	2010-10-21 07:00:00	142.0	361
+Chicago	New York	2010-10-21	2010-10-21 07:00:00	77.0	897
 Chicago	New York	2010-10-22	2010-10-22 07:00:00	2.0	361
+Chicago	New York	2010-10-22	2010-10-22 07:00:00	24.0	897
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	1.0	650
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	3.0	384
+Chicago	New York	2010-10-23	2010-10-23 07:00:00	32.0	426
 Chicago	New York	2010-10-24	2010-10-24 07:00:00	10.0	361
+Chicago	New York	2010-10-24	2010-10-24 07:00:00	113.0	897
+Chicago	New York	2010-10-25	2010-10-25 07:00:00	-1.0	897
 Chicago	New York	2010-10-25	2010-10-25 07:00:00	31.0	361
+Chicago	New York	2010-10-26	2010-10-26 07:00:00	0.0	897
 Chicago	New York	2010-10-26	2010-10-26 07:00:00	12.0	361
+Chicago	New York	2010-10-27	2010-10-27 07:00:00	-11.0	897
 Chicago	New York	2010-10-27	2010-10-27 07:00:00	148.0	361
 Chicago	New York	2010-10-28	2010-10-28 07:00:00	2.0	361
 Chicago	New York	2010-10-29	2010-10-29 07:00:00	-12.0	361
-Chicago	New York	2010-10-31	2010-10-31 07:00:00	-25.0	361
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	3.0	384
-Chicago	New York	2010-10-30	2010-10-30 07:00:00	-6.0	384
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	32.0	426
 Chicago	New York	2010-10-30	2010-10-30 07:00:00	-10.0	426
-Chicago	New York	2010-10-23	2010-10-23 07:00:00	1.0	650
 Chicago	New York	2010-10-30	2010-10-30 07:00:00	-5.0	650
-Chicago	New York	2010-10-20	2010-10-20 07:00:00	24.0	897
-Chicago	New York	2010-10-21	2010-10-21 07:00:00	77.0	897
-Chicago	New York	2010-10-22	2010-10-22 07:00:00	24.0	897
-Chicago	New York	2010-10-24	2010-10-24 07:00:00	113.0	897
-Chicago	New York	2010-10-25	2010-10-25 07:00:00	-1.0	897
-Chicago	New York	2010-10-26	2010-10-26 07:00:00	0.0	897
-Chicago	New York	2010-10-27	2010-10-27 07:00:00	-11.0	897
+Chicago	New York	2010-10-30	2010-10-30 07:00:00	-6.0	384
+Chicago	New York	2010-10-31	2010-10-31 07:00:00	-25.0	361
 PREHOOK: query: explain vectorization expression
 select fl_date, count(*) from flights_tiny_orc group by fl_date
 PREHOOK: type: QUERY
@@ -537,16 +537,16 @@ POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@flights_tiny_orc
 #### A masked pattern was here ####
+2010-10-20	11
 2010-10-21	12
 2010-10-22	11
-2010-10-29	12
-2010-10-20	11
 2010-10-23	12
 2010-10-24	12
 2010-10-25	12
 2010-10-26	13
 2010-10-27	11
 2010-10-28	12
+2010-10-29	12
 2010-10-30	11
 2010-10-31	8
 PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date (
@@ -690,142 +690,142 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31
 #### A masked pattern was here ####
-Baltimore	New York	2010-10-26 07:00:00	-22.0	1064	2010-10-26
-Baltimore	New York	2010-10-26 07:00:00	123.0	1142	2010-10-26
-Baltimore	New York	2010-10-26 07:00:00	90.0	1599	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	12.0	361	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	29.0	1531	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	-17.0	1610	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	6.0	3198	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	4.0	2630	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	-27.0	2646	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	-11.0	2662	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	13.0	3014	2010-10-26
-Washington	New York	2010-10-26 07:00:00	4.0	7291	2010-10-26
-Baltimore	New York	2010-10-21 07:00:00	17.0	1064	2010-10-21
-Baltimore	New York	2010-10-21 07:00:00	105.0	1142	2010-10-21
-Baltimore	New York	2010-10-21 07:00:00	28.0	1599	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	142.0	361	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	53.0	1531	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	-5.0	1610	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	51.0	3198	2010-10-21
-Cleveland	New York	2010-10-21 07:00:00	3.0	2630	2010-10-21
-Cleveland	New York	2010-10-21 07:00:00	29.0	2646	2010-10-21
-Cleveland	New York	2010-10-21 07:00:00	72.0	3014	2010-10-21
-Washington	New York	2010-10-21 07:00:00	22.0	7291	2010-10-21
-Baltimore	New York	2010-10-25 07:00:00	-25.0	1064	2010-10-25
-Baltimore	New York	2010-10-25 07:00:00	92.0	1142	2010-10-25
-Baltimore	New York	2010-10-25 07:00:00	106.0	1599	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	31.0	361	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	43.0	1531	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	6.0	1610	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	-16.0	3198	2010-10-25
-Cleveland	New York	2010-10-25 07:00:00	-4.0	2630	2010-10-25
-Cleveland	New York	2010-10-25 07:00:00	81.0	2646	2010-10-25
-Cleveland	New York	2010-10-25 07:00:00	42.0	3014	2010-10-25
-Washington	New York	2010-10-25 07:00:00	9.0	7291	2010-10-25
-Baltimore	New York	2010-10-24 07:00:00	12.0	1599	2010-10-24
-Baltimore	New York	2010-10-24 07:00:00	20.0	2571	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	10.0	361	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	-5.0	1531	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	-17.0	1610	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	-3.0	3198	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	5.0	2254	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	-11.0	2630	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	-20.0	2646	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	-9.0	3014	2010-10-24
-Washington	New York	2010-10-24 07:00:00	-26.0	7282	2010-10-24
-Baltimore	New York	2010-10-23 07:00:00	18.0	272	2010-10-23
+Baltimore	New York	2010-10-20 07:00:00	-30.0	1064	2010-10-20
+Baltimore	New York	2010-10-20 07:00:00	23.0	1142	2010-10-20
+Baltimore	New York	2010-10-20 07:00:00	6.0	1599	2010-10-20
+Baltimore	New York	2010-10-21 07:00:00	105.0	1142	2010-10-21
+Baltimore	New York	2010-10-21 07:00:00	17.0	1064	2010-10-21
+Baltimore	New York	2010-10-21 07:00:00	28.0	1599	2010-10-21
+Baltimore	New York	2010-10-22 07:00:00	-12.0	1064	2010-10-22
+Baltimore	New York	2010-10-22 07:00:00	18.0	1599	2010-10-22
+Baltimore	New York	2010-10-22 07:00:00	54.0	1142	2010-10-22
 Baltimore	New York	2010-10-23 07:00:00	-10.0	1805	2010-10-23
+Baltimore	New York	2010-10-23 07:00:00	18.0	272	2010-10-23
 Baltimore	New York	2010-10-23 07:00:00	6.0	3171	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	11.0	3085	2010-10-23
-Cleveland	New York	2010-10-23 07:00:00	-21.0	2932	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-25.0	5832	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-21.0	5904	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-18.0	5917	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-16.0	7274	2010-10-23
-Baltimore	New York	2010-10-29 07:00:00	-24.0	1064	2010-10-29
-Baltimore	New York	2010-10-29 07:00:00	21.0	1142	2010-10-29
-Baltimore	New York	2010-10-29 07:00:00	-2.0	1599	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-12.0	361	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-11.0	897	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	15.0	1531	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-18.0	1610	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-4.0	3198	2010-10-29
-Cleveland	New York	2010-10-29 07:00:00	-4.0	2630	2010-10-29
-Cleveland	New York	2010-10-29 07:00:00	-19.0	2646	2010-10-29
-Cleveland	New York	2010-10-29 07:00:00	-12.0	3014	2010-10-29
-Washington	New York	2010-10-29 07:00:00	1.0	7291	2010-10-29
+Baltimore	New York	2010-10-24 07:00:00	12.0	1599	2010-10-24
+Baltimore	New York	2010-10-24 07:00:00	20.0	2571	2010-10-24
+Baltimore	New York	2010-10-25 07:00:00	-25.0	1064	2010-10-25
+Baltimore	New York	2010-10-25 07:00:00	106.0	1599	2010-10-25
+Baltimore	New York	2010-10-25 07:00:00	92.0	1142	2010-10-25
+Baltimore	New York	2010-10-26 07:00:00	-22.0	1064	2010-10-26
+Baltimore	New York	2010-10-26 07:00:00	123.0	1142	2010-10-26
+Baltimore	New York	2010-10-26 07:00:00	90.0	1599	2010-10-26
 Baltimore	New York	2010-10-27 07:00:00	-18.0	1064	2010-10-27
 Baltimore	New York	2010-10-27 07:00:00	49.0	1142	2010-10-27
 Baltimore	New York	2010-10-27 07:00:00	92.0	1599	2010-10-27
-Chicago	New York	2010-10-27 07:00:00	148.0	361	2010-10-27
+Baltimore	New York	2010-10-28 07:00:00	-14.0	1142	2010-10-28
+Baltimore	New York	2010-10-28 07:00:00	-14.0	1599	2010-10-28
+Baltimore	New York	2010-10-28 07:00:00	-4.0	1064	2010-10-28
+Baltimore	New York	2010-10-29 07:00:00	-2.0	1599	2010-10-29
+Baltimore	New York	2010-10-29 07:00:00	-24.0	1064	2010-10-29
+Baltimore	New York	2010-10-29 07:00:00	21.0	1142	2010-10-29
+Baltimore	New York	2010-10-30 07:00:00	-1.0	1805	2010-10-30
+Baltimore	New York	2010-10-30 07:00:00	14.0	272	2010-10-30
+Baltimore	New York	2010-10-30 07:00:00	5.0	3171	2010-10-30
+Baltimore	New York	2010-10-31 07:00:00	-1.0	1599	2010-10-31
+Baltimore	New York	2010-10-31 07:00:00	-14.0	2571	2010-10-31
+Chicago	New York	2010-10-20 07:00:00	-2.0	3198	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	-6.0	1610	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	15.0	1531	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	42.0	361	2010-10-20
+Chicago	New York	2010-10-21 07:00:00	-5.0	1610	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	142.0	361	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	51.0	3198	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	53.0	1531	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
+Chicago	New York	2010-10-22 07:00:00	-11.0	3198	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	-6.0	1610	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	16.0	1531	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	2.0	361	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
+Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	11.0	3085	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
+Chicago	New York	2010-10-24 07:00:00	-17.0	1610	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	-3.0	3198	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	-5.0	1531	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	10.0	361	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
+Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	-16.0	3198	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	31.0	361	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	43.0	1531	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	6.0	1610	2010-10-25
+Chicago	New York	2010-10-26 07:00:00	-17.0	1610	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	12.0	361	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	29.0	1531	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	6.0	3198	2010-10-26
 Chicago	New York	2010-10-27 07:00:00	-11.0	897	2010-10-27
+Chicago	New York	2010-10-27 07:00:00	148.0	361	2010-10-27
+Chicago	New York	2010-10-27 07:00:00	21.0	3198	2010-10-27
 Chicago	New York	2010-10-27 07:00:00	70.0	1531	2010-10-27
 Chicago	New York	2010-10-27 07:00:00	8.0	1610	2010-10-27
-Chicago	New York	2010-10-27 07:00:00	21.0	3198	2010-10-27
-Cleveland	New York	2010-10-27 07:00:00	16.0	2630	2010-10-27
-Cleveland	New York	2010-10-27 07:00:00	27.0	3014	2010-10-27
-Washington	New York	2010-10-27 07:00:00	26.0	7291	2010-10-27
-Baltimore	New York	2010-10-28 07:00:00	-4.0	1064	2010-10-28
-Baltimore	New York	2010-10-28 07:00:00	-14.0	1142	2010-10-28
-Baltimore	New York	2010-10-28 07:00:00	-14.0	1599	2010-10-28
+Chicago	New York	2010-10-28 07:00:00	-11.0	1531	2010-10-28
+Chicago	New York	2010-10-28 07:00:00	-18.0	3198	2010-10-28
 Chicago	New York	2010-10-28 07:00:00	2.0	361	2010-10-28
 Chicago	New York	2010-10-28 07:00:00	2.0	897	2010-10-28
-Chicago	New York	2010-10-28 07:00:00	-11.0	1531	2010-10-28
 Chicago	New York	2010-10-28 07:00:00	3.0	1610	2010-10-28
-Chicago	New York	2010-10-28 07:00:00	-18.0	3198	2010-10-28
-Cleveland	New York	2010-10-28 07:00:00	3.0	2630	2010-10-28
-Cleveland	New York	2010-10-28 07:00:00	-6.0	2646	2010-10-28
-Cleveland	New York	2010-10-28 07:00:00	1.0	3014	2010-10-28
-Washington	New York	2010-10-28 07:00:00	45.0	7291	2010-10-28
-Baltimore	New York	2010-10-30 07:00:00	14.0	272	2010-10-30
-Baltimore	New York	2010-10-30 07:00:00	-1.0	1805	2010-10-30
-Baltimore	New York	2010-10-30 07:00:00	5.0	3171	2010-10-30
-Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
+Chicago	New York	2010-10-29 07:00:00	-11.0	897	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	-12.0	361	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	-18.0	1610	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	-4.0	3198	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	15.0	1531	2010-10-29
 Chicago	New York	2010-10-30 07:00:00	-10.0	426	2010-10-30
-Chicago	New York	2010-10-30 07:00:00	-5.0	650	2010-10-30
 Chicago	New York	2010-10-30 07:00:00	-5.0	3085	2010-10-30
-Cleveland	New York	2010-10-30 07:00:00	-23.0	2018	2010-10-30
-Cleveland	New York	2010-10-30 07:00:00	-12.0	2932	2010-10-30
-Washington	New York	2010-10-30 07:00:00	-27.0	5904	2010-10-30
-Washington	New York	2010-10-30 07:00:00	-16.0	5917	2010-10-30
-Baltimore	New York	2010-10-20 07:00:00	-30.0	1064	2010-10-20
-Baltimore	New York	2010-10-20 07:00:00	23.0	1142	2010-10-20
-Baltimore	New York	2010-10-20 07:00:00	6.0	1599	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	42.0	361	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	15.0	1531	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	-6.0	1610	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	-2.0	3198	2010-10-20
-Cleveland	New York	2010-10-20 07:00:00	-8.0	2630	2010-10-20
+Chicago	New York	2010-10-30 07:00:00	-5.0	650	2010-10-30
+Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
+Chicago	New York	2010-10-31 07:00:00	-15.0	3198	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-18.0	897	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-22.0	1610	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-4.0	1531	2010-10-31
 Cleveland	New York	2010-10-20 07:00:00	-15.0	3014	2010-10-20
-Washington	New York	2010-10-20 07:00:00	-2.0	7291	2010-10-20
-Baltimore	New York	2010-10-22 07:00:00	-12.0	1064	2010-10-22
-Baltimore	New York	2010-10-22 07:00:00	54.0	1142	2010-10-22
-Baltimore	New York	2010-10-22 07:00:00	18.0	1599	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	2.0	361	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	16.0	1531	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	-6.0	1610	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	-11.0	3198	2010-10-22
-Cleveland	New York	2010-10-22 07:00:00	1.0	2630	2010-10-22
+Cleveland	New York	2010-10-20 07:00:00	-8.0	2630	2010-10-20
+Cleveland	New York	2010-10-21 07:00:00	29.0	2646	2010-10-21
+Cleveland	New York	2010-10-21 07:00:00	3.0	2630	2010-10-21
+Cleveland	New York	2010-10-21 07:00:00	72.0	3014	2010-10-21
 Cleveland	New York	2010-10-22 07:00:00	-25.0	2646	2010-10-22
 Cleveland	New York	2010-10-22 07:00:00	-3.0	3014	2010-10-22
-Baltimore	New York	2010-10-31 07:00:00	-1.0	1599	2010-10-31
-Baltimore	New York	2010-10-31 07:00:00	-14.0	2571	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-18.0	897	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-4.0	1531	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-22.0	1610	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-15.0	3198	2010-10-31
+Cleveland	New York	2010-10-22 07:00:00	1.0	2630	2010-10-22
+Cleveland	New York	2010-10-23 07:00:00	-21.0	2932	2010-10-23
+Cleveland	New York	2010-10-24 07:00:00	-11.0	2630	2010-10-24
+Cleveland	New York	2010-10-24 07:00:00	-20.0	2646	2010-10-24
+Cleveland	New York	2010-10-24 07:00:00	-9.0	3014	2010-10-24
+Cleveland	New York	2010-10-24 07:00:00	5.0	2254	2010-10-24
+Cleveland	New York	2010-10-25 07:00:00	-4.0	2630	2010-10-25
+Cleveland	New York	2010-10-25 07:00:00	42.0	3014	2010-10-25
+Cleveland	New York	2010-10-25 07:00:00	81.0	2646	2010-10-25
+Cleveland	New York	2010-10-26 07:00:00	-11.0	2662	2010-10-26
+Cleveland	New York	2010-10-26 07:00:00	-27.0	2646	2010-10-26
+Cleveland	New York	2010-10-26 07:00:00	13.0	3014	2010-10-26
+Cleveland	New York	2010-10-26 07:00:00	4.0	2630	2010-10-26
+Cleveland	New York	2010-10-27 07:00:00	16.0	2630	2010-10-27
+Cleveland	New York	2010-10-27 07:00:00	27.0	3014	2010-10-27
+Cleveland	New York	2010-10-28 07:00:00	-6.0	2646	2010-10-28
+Cleveland	New York	2010-10-28 07:00:00	1.0	3014	2010-10-28
+Cleveland	New York	2010-10-28 07:00:00	3.0	2630	2010-10-28
+Cleveland	New York	2010-10-29 07:00:00	-12.0	3014	2010-10-29
+Cleveland	New York	2010-10-29 07:00:00	-19.0	2646	2010-10-29
+Cleveland	New York	2010-10-29 07:00:00	-4.0	2630	2010-10-29
+Cleveland	New York	2010-10-30 07:00:00	-12.0	2932	2010-10-30
+Cleveland	New York	2010-10-30 07:00:00	-23.0	2018	2010-10-30
+Washington	New York	2010-10-20 07:00:00	-2.0	7291	2010-10-20
+Washington	New York	2010-10-21 07:00:00	22.0	7291	2010-10-21
+Washington	New York	2010-10-23 07:00:00	-16.0	7274	2010-10-23
+Washington	New York	2010-10-23 07:00:00	-18.0	5917	2010-10-23
+Washington	New York	2010-10-23 07:00:00	-21.0	5904	2010-10-23
+Washington	New York	2010-10-23 07:00:00	-25.0	5832	2010-10-23
+Washington	New York	2010-10-24 07:00:00	-26.0	7282	2010-10-24
+Washington	New York	2010-10-25 07:00:00	9.0	7291	2010-10-25
+Washington	New York	2010-10-26 07:00:00	4.0	7291	2010-10-26
+Washington	New York	2010-10-27 07:00:00	26.0	7291	2010-10-27
+Washington	New York	2010-10-28 07:00:00	45.0	7291	2010-10-28
+Washington	New York	2010-10-29 07:00:00	1.0	7291	2010-10-29
+Washington	New York	2010-10-30 07:00:00	-16.0	5917	2010-10-30
+Washington	New York	2010-10-30 07:00:00	-27.0	5904	2010-10-30
 Washington	New York	2010-10-31 07:00:00	-18.0	7282	2010-10-31
 PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25
 PREHOOK: type: QUERY
@@ -861,29 +861,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31
 #### A masked pattern was here ####
 Baltimore	New York	2010-10-23 07:00:00	18.0	272	2010-10-23
 Baltimore	New York	2010-10-30 07:00:00	14.0	272	2010-10-30
+Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
 Chicago	New York	2010-10-20 07:00:00	42.0	361	2010-10-20
 Chicago	New York	2010-10-21 07:00:00	142.0	361	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
 Chicago	New York	2010-10-22 07:00:00	2.0	361	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
+Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
 Chicago	New York	2010-10-24 07:00:00	10.0	361	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
+Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
 Chicago	New York	2010-10-25 07:00:00	31.0	361	2010-10-25
+Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
 Chicago	New York	2010-10-26 07:00:00	12.0	361	2010-10-26
+Chicago	New York	2010-10-27 07:00:00	-11.0	897	2010-10-27
 Chicago	New York	2010-10-27 07:00:00	148.0	361	2010-10-27
 Chicago	New York	2010-10-28 07:00:00	2.0	361	2010-10-28
 Chicago	New York	2010-10-29 07:00:00	-12.0	361	2010-10-29
-Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
-Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
-Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
-Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
 Chicago	New York	2010-10-30 07:00:00	-10.0	426	2010-10-30
-Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
 Chicago	New York	2010-10-30 07:00:00	-5.0	650	2010-10-30
-Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
-Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
-Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
-Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
-Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
-Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
-Chicago	New York	2010-10-27 07:00:00	-11.0	897	2010-10-27
+Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
+Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
 PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date
 PREHOOK: type: QUERY
 PREHOOK: Input: default@flights_tiny_orc_partitioned_date
@@ -916,16 +916,16 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31
 #### A masked pattern was here ####
+2010-10-20	11
 2010-10-21	12
 2010-10-22	11
-2010-10-29	12
-2010-10-20	11
 2010-10-23	12
 2010-10-24	12
 2010-10-25	12
 2010-10-26	13
 2010-10-27	11
 2010-10-28	12
+2010-10-29	12
 2010-10-30	11
 2010-10-31	8
 PREHOOK: query: explain vectorization expression
@@ -979,7 +979,7 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: flights_tiny_orc_partitioned_date
-                  Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL
                   TableScan Vectorization:
                       native: true
                   Select Operator
@@ -989,13 +989,13 @@ STAGE PLANS:
                         className: VectorSelectOperator
                         native: true
                         projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
-                    Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
                       File Sink Vectorization:
                           className: VectorFileSinkOperator
                           native: false
-                      Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1050,142 +1050,142 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31
 #### A masked pattern was here ####
-Baltimore	New York	2010-10-26 07:00:00	-22.0	1064	2010-10-26
-Baltimore	New York	2010-10-26 07:00:00	123.0	1142	2010-10-26
-Baltimore	New York	2010-10-26 07:00:00	90.0	1599	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	12.0	361	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	29.0	1531	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	-17.0	1610	2010-10-26
-Chicago	New York	2010-10-26 07:00:00	6.0	3198	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	4.0	2630	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	-27.0	2646	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	-11.0	2662	2010-10-26
-Cleveland	New York	2010-10-26 07:00:00	13.0	3014	2010-10-26
-Washington	New York	2010-10-26 07:00:00	4.0	7291	2010-10-26
-Baltimore	New York	2010-10-21 07:00:00	17.0	1064	2010-10-21
+Baltimore	New York	2010-10-20 07:00:00	-30.0	1064	2010-10-20
+Baltimore	New York	2010-10-20 07:00:00	23.0	1142	2010-10-20
+Baltimore	New York	2010-10-20 07:00:00	6.0	1599	2010-10-20
 Baltimore	New York	2010-10-21 07:00:00	105.0	1142	2010-10-21
+Baltimore	New York	2010-10-21 07:00:00	17.0	1064	2010-10-21
 Baltimore	New York	2010-10-21 07:00:00	28.0	1599	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	142.0	361	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	53.0	1531	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	-5.0	1610	2010-10-21
-Chicago	New York	2010-10-21 07:00:00	51.0	3198	2010-10-21
-Cleveland	New York	2010-10-21 07:00:00	3.0	2630	2010-10-21
-Cleveland	New York	2010-10-21 07:00:00	29.0	2646	2010-10-21
-Cleveland	New York	2010-10-21 07:00:00	72.0	3014	2010-10-21
-Washington	New York	2010-10-21 07:00:00	22.0	7291	2010-10-21
-Baltimore	New York	2010-10-25 07:00:00	-25.0	1064	2010-10-25
-Baltimore	New York	2010-10-25 07:00:00	92.0	1142	2010-10-25
-Baltimore	New York	2010-10-25 07:00:00	106.0	1599	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	31.0	361	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	43.0	1531	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	6.0	1610	2010-10-25
-Chicago	New York	2010-10-25 07:00:00	-16.0	3198	2010-10-25
-Cleveland	New York	2010-10-25 07:00:00	-4.0	2630	2010-10-25
-Cleveland	New York	2010-10-25 07:00:00	81.0	2646	2010-10-25
-Cleveland	New York	2010-10-25 07:00:00	42.0	3014	2010-10-25
-Washington	New York	2010-10-25 07:00:00	9.0	7291	2010-10-25
-Baltimore	New York	2010-10-24 07:00:00	12.0	1599	2010-10-24
-Baltimore	New York	2010-10-24 07:00:00	20.0	2571	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	10.0	361	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	-5.0	1531	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	-17.0	1610	2010-10-24
-Chicago	New York	2010-10-24 07:00:00	-3.0	3198	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	5.0	2254	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	-11.0	2630	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	-20.0	2646	2010-10-24
-Cleveland	New York	2010-10-24 07:00:00	-9.0	3014	2010-10-24
-Washington	New York	2010-10-24 07:00:00	-26.0	7282	2010-10-24
-Baltimore	New York	2010-10-23 07:00:00	18.0	272	2010-10-23
+Baltimore	New York	2010-10-22 07:00:00	-12.0	1064	2010-10-22
+Baltimore	New York	2010-10-22 07:00:00	18.0	1599	2010-10-22
+Baltimore	New York	2010-10-22 07:00:00	54.0	1142	2010-10-22
 Baltimore	New York	2010-10-23 07:00:00	-10.0	1805	2010-10-23
+Baltimore	New York	2010-10-23 07:00:00	18.0	272	2010-10-23
 Baltimore	New York	2010-10-23 07:00:00	6.0	3171	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
-Chicago	New York	2010-10-23 07:00:00	11.0	3085	2010-10-23
-Cleveland	New York	2010-10-23 07:00:00	-21.0	2932	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-25.0	5832	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-21.0	5904	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-18.0	5917	2010-10-23
-Washington	New York	2010-10-23 07:00:00	-16.0	7274	2010-10-23
-Baltimore	New York	2010-10-29 07:00:00	-24.0	1064	2010-10-29
-Baltimore	New York	2010-10-29 07:00:00	21.0	1142	2010-10-29
-Baltimore	New York	2010-10-29 07:00:00	-2.0	1599	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-12.0	361	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-11.0	897	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	15.0	1531	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-18.0	1610	2010-10-29
-Chicago	New York	2010-10-29 07:00:00	-4.0	3198	2010-10-29
-Cleveland	New York	2010-10-29 07:00:00	-4.0	2630	2010-10-29
-Cleveland	New York	2010-10-29 07:00:00	-19.0	2646	2010-10-29
-Cleveland	New York	2010-10-29 07:00:00	-12.0	3014	2010-10-29
-Washington	New York	2010-10-29 07:00:00	1.0	7291	2010-10-29
+Baltimore	New York	2010-10-24 07:00:00	12.0	1599	2010-10-24
+Baltimore	New York	2010-10-24 07:00:00	20.0	2571	2010-10-24
+Baltimore	New York	2010-10-25 07:00:00	-25.0	1064	2010-10-25
+Baltimore	New York	2010-10-25 07:00:00	106.0	1599	2010-10-25
+Baltimore	New York	2010-10-25 07:00:00	92.0	1142	2010-10-25
+Baltimore	New York	2010-10-26 07:00:00	-22.0	1064	2010-10-26
+Baltimore	New York	2010-10-26 07:00:00	123.0	1142	2010-10-26
+Baltimore	New York	2010-10-26 07:00:00	90.0	1599	2010-10-26
 Baltimore	New York	2010-10-27 07:00:00	-18.0	1064	2010-10-27
 Baltimore	New York	2010-10-27 07:00:00	49.0	1142	2010-10-27
 Baltimore	New York	2010-10-27 07:00:00	92.0	1599	2010-10-27
-Chicago	New York	2010-10-27 07:00:00	148.0	361	2010-10-27
+Baltimore	New York	2010-10-28 07:00:00	-14.0	1142	2010-10-28
+Baltimore	New York	2010-10-28 07:00:00	-14.0	1599	2010-10-28
+Baltimore	New York	2010-10-28 07:00:00	-4.0	1064	2010-10-28
+Baltimore	New York	2010-10-29 07:00:00	-2.0	1599	2010-10-29
+Baltimore	New York	2010-10-29 07:00:00	-24.0	1064	2010-10-29
+Baltimore	New York	2010-10-29 07:00:00	21.0	1142	2010-10-29
+Baltimore	New York	2010-10-30 07:00:00	-1.0	1805	2010-10-30
+Baltimore	New York	2010-10-30 07:00:00	14.0	272	2010-10-30
+Baltimore	New York	2010-10-30 07:00:00	5.0	3171	2010-10-30
+Baltimore	New York	2010-10-31 07:00:00	-1.0	1599	2010-10-31
+Baltimore	New York	2010-10-31 07:00:00	-14.0	2571	2010-10-31
+Chicago	New York	2010-10-20 07:00:00	-2.0	3198	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	-6.0	1610	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	15.0	1531	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
+Chicago	New York	2010-10-20 07:00:00	42.0	361	2010-10-20
+Chicago	New York	2010-10-21 07:00:00	-5.0	1610	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	142.0	361	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	51.0	3198	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	53.0	1531	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
+Chicago	New York	2010-10-22 07:00:00	-11.0	3198	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	-6.0	1610	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	16.0	1531	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	2.0	361	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
+Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	11.0	3085	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
+Chicago	New York	2010-10-24 07:00:00	-17.0	1610	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	-3.0	3198	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	-5.0	1531	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	10.0	361	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
+Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	-16.0	3198	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	31.0	361	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	43.0	1531	2010-10-25
+Chicago	New York	2010-10-25 07:00:00	6.0	1610	2010-10-25
+Chicago	New York	2010-10-26 07:00:00	-17.0	1610	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	12.0	361	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	29.0	1531	2010-10-26
+Chicago	New York	2010-10-26 07:00:00	6.0	3198	2010-10-26
 Chicago	New York	2010-10-27 07:00:00	-11.0	897	2010-10-27
+Chicago	New York	2010-10-27 07:00:00	148.0	361	2010-10-27
+Chicago	New York	2010-10-27 07:00:00	21.0	3198	2010-10-27
 Chicago	New York	2010-10-27 07:00:00	70.0	1531	2010-10-27
 Chicago	New York	2010-10-27 07:00:00	8.0	1610	2010-10-27
-Chicago	New York	2010-10-27 07:00:00	21.0	3198	2010-10-27
-Cleveland	New York	2010-10-27 07:00:00	16.0	2630	2010-10-27
-Cleveland	New York	2010-10-27 07:00:00	27.0	3014	2010-10-27
-Washington	New York	2010-10-27 07:00:00	26.0	7291	2010-10-27
-Baltimore	New York	2010-10-28 07:00:00	-4.0	1064	2010-10-28
-Baltimore	New York	2010-10-28 07:00:00	-14.0	1142	2010-10-28
-Baltimore	New York	2010-10-28 07:00:00	-14.0	1599	2010-10-28
+Chicago	New York	2010-10-28 07:00:00	-11.0	1531	2010-10-28
+Chicago	New York	2010-10-28 07:00:00	-18.0	3198	2010-10-28
 Chicago	New York	2010-10-28 07:00:00	2.0	361	2010-10-28
 Chicago	New York	2010-10-28 07:00:00	2.0	897	2010-10-28
-Chicago	New York	2010-10-28 07:00:00	-11.0	1531	2010-10-28
 Chicago	New York	2010-10-28 07:00:00	3.0	1610	2010-10-28
-Chicago	New York	2010-10-28 07:00:00	-18.0	3198	2010-10-28
-Cleveland	New York	2010-10-28 07:00:00	3.0	2630	2010-10-28
-Cleveland	New York	2010-10-28 07:00:00	-6.0	2646	2010-10-28
-Cleveland	New York	2010-10-28 07:00:00	1.0	3014	2010-10-28
-Washington	New York	2010-10-28 07:00:00	45.0	7291	2010-10-28
-Baltimore	New York	2010-10-30 07:00:00	14.0	272	2010-10-30
-Baltimore	New York	2010-10-30 07:00:00	-1.0	1805	2010-10-30
-Baltimore	New York	2010-10-30 07:00:00	5.0	3171	2010-10-30
-Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
+Chicago	New York	2010-10-29 07:00:00	-11.0	897	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	-12.0	361	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	-18.0	1610	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	-4.0	3198	2010-10-29
+Chicago	New York	2010-10-29 07:00:00	15.0	1531	2010-10-29
 Chicago	New York	2010-10-30 07:00:00	-10.0	426	2010-10-30
-Chicago	New York	2010-10-30 07:00:00	-5.0	650	2010-10-30
 Chicago	New York	2010-10-30 07:00:00	-5.0	3085	2010-10-30
-Cleveland	New York	2010-10-30 07:00:00	-23.0	2018	2010-10-30
-Cleveland	New York	2010-10-30 07:00:00	-12.0	2932	2010-10-30
-Washington	New York	2010-10-30 07:00:00	-27.0	5904	2010-10-30
-Washington	New York	2010-10-30 07:00:00	-16.0	5917	2010-10-30
-Baltimore	New York	2010-10-20 07:00:00	-30.0	1064	2010-10-20
-Baltimore	New York	2010-10-20 07:00:00	23.0	1142	2010-10-20
-Baltimore	New York	2010-10-20 07:00:00	6.0	1599	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	42.0	361	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	15.0	1531	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	-6.0	1610	2010-10-20
-Chicago	New York	2010-10-20 07:00:00	-2.0	3198	2010-10-20
-Cleveland	New York	2010-10-20 07:00:00	-8.0	2630	2010-10-20
+Chicago	New York	2010-10-30 07:00:00	-5.0	650	2010-10-30
+Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
+Chicago	New York	2010-10-31 07:00:00	-15.0	3198	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-18.0	897	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-22.0	1610	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
+Chicago	New York	2010-10-31 07:00:00	-4.0	1531	2010-10-31
 Cleveland	New York	2010-10-20 07:00:00	-15.0	3014	2010-10-20
-Washington	New York	2010-10-20 07:00:00	-2.0	7291	2010-10-20
-Baltimore	New York	2010-10-22 07:00:00	-12.0	1064	2010-10-22
-Baltimore	New York	2010-10-22 07:00:00	54.0	1142	2010-10-22
-Baltimore	New York	2010-10-22 07:00:00	18.0	1599	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	2.0	361	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	16.0	1531	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	-6.0	1610	2010-10-22
-Chicago	New York	2010-10-22 07:00:00	-11.0	3198	2010-10-22
-Cleveland	New York	2010-10-22 07:00:00	1.0	2630	2010-10-22
+Cleveland	New York	2010-10-20 07:00:00	-8.0	2630	2010-10-20
+Cleveland	New York	2010-10-21 07:00:00	29.0	2646	2010-10-21
+Cleveland	New York	2010-10-21 07:00:00	3.0	2630	2010-10-21
+Cleveland	New York	2010-10-21 07:00:00	72.0	3014	2010-10-21
 Cleveland	New York	2010-10-22 07:00:00	-25.0	2646	2010-10-22
 Cleveland	New York	2010-10-22 07:00:00	-3.0	3014	2010-10-22
-Baltimore	New York	2010-10-31 07:00:00	-1.0	1599	2010-10-31
-Baltimore	New York	2010-10-31 07:00:00	-14.0	2571	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-18.0	897	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-4.0	1531	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-22.0	1610	2010-10-31
-Chicago	New York	2010-10-31 07:00:00	-15.0	3198	2010-10-31
+Cleveland	New York	2010-10-22 07:00:00	1.0	2630	2010-10-22
+Cleveland	New York	2010-10-23 07:00:00	-21.0	2932	2010-10-23
+Cleveland	New York	2010-10-24 07:00:00	-11.0	2630	2010-10-24
+Cleveland	New York	2010-10-24 07:00:00	-20.0	2646	2010-10-24
+Cleveland	New York	2010-10-24 07:00:00	-9.0	3014	2010-10-24
+Cleveland	New York	2010-10-24 07:00:00	5.0	2254	2010-10-24
+Cleveland	New York	2010-10-25 07:00:00	-4.0	2630	2010-10-25
+Cleveland	New York	2010-10-25 07:00:00	42.0	3014	2010-10-25
+Cleveland	New York	2010-10-25 07:00:00	81.0	2646	2010-10-25
+Cleveland	New York	2010-10-26 07:00:00	-11.0	2662	2010-10-26
+Cleveland	New York	2010-10-26 07:00:00	-27.0	2646	2010-10-26
+Cleveland	New York	2010-10-26 07:00:00	13.0	3014	2010-10-26
+Cleveland	New York	2010-10-26 07:00:00	4.0	2630	2010-10-26
+Cleveland	New York	2010-10-27 07:00:00	16.0	2630	2010-10-27
+Cleveland	New York	2010-10-27 07:00:00	27.0	3014	2010-10-27
+Cleveland	New York	2010-10-28 07:00:00	-6.0	2646	2010-10-28
+Cleveland	New York	2010-10-28 07:00:00	1.0	3014	2010-10-28
+Cleveland	New York	2010-10-28 07:00:00	3.0	2630	2010-10-28
+Cleveland	New York	2010-10-29 07:00:00	-12.0	3014	2010-10-29
+Cleveland	New York	2010-10-29 07:00:00	-19.0	2646	2010-10-29
+Cleveland	New York	2010-10-29 07:00:00	-4.0	2630	2010-10-29
+Cleveland	New York	2010-10-30 07:00:00	-12.0	2932	2010-10-30
+Cleveland	New York	2010-10-30 07:00:00	-23.0	2018	2010-10-30
+Washington	New York	2010-10-20 07:00:00	-2.0	7291	2010-10-20
+Washington	New York	2010-10-21 07:00:00	22.0	7291	2010-10-21
+Washington	New York	2010-10-23 07:00:00	-16.0	7274	2010-10-23
+Washington	New York	2010-10-23 07:00:00	-18.0	5917	2010-10-23
+Washington	New York	2010-10-23 07:00:00	-21.0	5904	2010-10-23
+Washington	New York	2010-10-23 07:00:00	-25.0	5832	2010-10-23
+Washington	New York	2010-10-24 07:00:00	-26.0	7282	2010-10-24
+Washington	New York	2010-10-25 07:00:00	9.0	7291	2010-10-25
+Washington	New York	2010-10-26 07:00:00	4.0	7291	2010-10-26
+Washington	New York	2010-10-27 07:00:00	26.0	7291	2010-10-27
+Washington	New York	2010-10-28 07:00:00	45.0	7291	2010-10-28
+Washington	New York	2010-10-29 07:00:00	1.0	7291	2010-10-29
+Washington	New York	2010-10-30 07:00:00	-16.0	5917	2010-10-30
+Washington	New York	2010-10-30 07:00:00	-27.0	5904	2010-10-30
 Washington	New York	2010-10-31 07:00:00	-18.0	7282	2010-10-31
 PREHOOK: query: explain vectorization expression
 select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25
@@ -1242,7 +1242,7 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: flights_tiny_orc_partitioned_date
-                  Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL
                   TableScan Vectorization:
                       native: true
                   Select Operator
@@ -1252,7 +1252,7 @@ STAGE PLANS:
                         className: VectorSelectOperator
                         native: true
                         projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
-                    Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL
                     Reduce Output Operator
                       key expressions: _col4 (type: int), _col5 (type: date)
                       sort order: ++
@@ -1260,7 +1260,7 @@ STAGE PLANS:
                           className: VectorReduceSinkObjectHashOperator
                           native: true
                           nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
-                      Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL
                       TopN Hash Memory Usage: 0.1
                       value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float)
             Execution mode: vectorized, llap
@@ -1290,13 +1290,13 @@ STAGE PLANS:
                     className: VectorSelectOperator
                     native: true
                     projectedOutputColumnNums: [2, 3, 4, 5, 0, 1]
-                Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL
                 Limit
                   Number of rows: 25
                   Limit Vectorization:
                       className: VectorLimitOperator
                       native: true
-                  Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL
                   Reduce Output Operator
                     key expressions: _col4 (type: int), _col5 (type: date)
                     sort order: ++
@@ -1304,7 +1304,7 @@ STAGE PLANS:
                         className: VectorReduceSinkObjectHashOperator
                         native: true
                         nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
-                    Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL
                     TopN Hash Memory Usage: 0.1
                     value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float)
         Reducer 3 
@@ -1323,19 +1323,19 @@ STAGE PLANS:
                     className: VectorSelectOperator
                     native: true
                     projectedOutputColumnNums: [2, 3, 4, 5, 0, 1]
-                Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL
                 Limit
                   Number of rows: 25
                   Limit Vectorization:
                       className: VectorLimitOperator
                       native: true
-                  Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL
                   File Output Operator
                     compressed: false
                     File Sink Vectorization:
                         className: VectorFileSinkOperator
                         native: false
-                    Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL
                     table:
                         input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                         output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1381,29 +1381,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31
 #### A masked pattern was here ####
 Baltimore	New York	2010-10-23 07:00:00	18.0	272	2010-10-23
 Baltimore	New York	2010-10-30 07:00:00	14.0	272	2010-10-30
+Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
 Chicago	New York	2010-10-20 07:00:00	42.0	361	2010-10-20
 Chicago	New York	2010-10-21 07:00:00	142.0	361	2010-10-21
+Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
 Chicago	New York	2010-10-22 07:00:00	2.0	361	2010-10-22
+Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
+Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
+Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
 Chicago	New York	2010-10-24 07:00:00	10.0	361	2010-10-24
+Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
+Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
 Chicago	New York	2010-10-25 07:00:00	31.0	361	2010-10-25
+Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
 Chicago	New York	2010-10-26 07:00:00	12.0	361	2010-10-26
+Chicago	New York	2010-10-27 07:00:00	-11.0	897	2010-10-27
 Chicago	New York	2010-10-27 07:00:00	148.0	361	2010-10-27
 Chicago	New York	2010-10-28 07:00:00	2.0	361	2010-10-28
 Chicago	New York	2010-10-29 07:00:00	-12.0	361	2010-10-29
-Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
-Chicago	New York	2010-10-23 07:00:00	3.0	384	2010-10-23
-Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
-Chicago	New York	2010-10-23 07:00:00	32.0	426	2010-10-23
 Chicago	New York	2010-10-30 07:00:00	-10.0	426	2010-10-30
-Chicago	New York	2010-10-23 07:00:00	1.0	650	2010-10-23
 Chicago	New York	2010-10-30 07:00:00	-5.0	650	2010-10-30
-Chicago	New York	2010-10-20 07:00:00	24.0	897	2010-10-20
-Chicago	New York	2010-10-21 07:00:00	77.0	897	2010-10-21
-Chicago	New York	2010-10-22 07:00:00	24.0	897	2010-10-22
-Chicago	New York	2010-10-24 07:00:00	113.0	897	2010-10-24
-Chicago	New York	2010-10-25 07:00:00	-1.0	897	2010-10-25
-Chicago	New York	2010-10-26 07:00:00	0.0	897	2010-10-26
-Chicago	New York	2010-10-27 07:00:00	-11.0	897	2010-10-27
+Chicago	New York	2010-10-30 07:00:00	-6.0	384	2010-10-30
+Chicago	New York	2010-10-31 07:00:00	-25.0	361	2010-10-31
 PREHOOK: query: explain vectorization expression
 select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date
 PREHOOK: type: QUERY
@@ -1576,16 +1576,16 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31
 #### A masked pattern was here ####
+2010-10-20	11
 2010-10-21	12
 2010-10-22	11
-2010-10-29	12
-2010-10-20	11
 2010-10-23	12
 2010-10-24	12
 2010-10-25	12
 2010-10-26	13
 2010-10-27	11
 2010-10-28	12
+2010-10-29	12
 2010-10-30	11
 2010-10-31	8
 PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp (
@@ -1729,142 +1729,142 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00
 #### A masked pattern was here ####
+Baltimore	New York	2010-10-20	-30.0	1064	2010-10-20 07:00:00
+Baltimore	New York	2010-10-20	23.0	1142	2010-10-20 07:00:00
+Baltimore	New York	2010-10-20	6.0	1599	2010-10-20 07:00:00
+Baltimore	New York	2010-10-21	105.0	1142	2010-10-21 07:00:00
+Baltimore	New York	2010-10-21	17.0	1064	2010-10-21 07:00:00
+Baltimore	New York	2010-10-21	28.0	1599	2010-10-21 07:00:00
+Baltimore	New York	2010-10-22	-12.0	1064	2010-10-22 07:00:00
+Baltimore	New York	2010-10-22	18.0	1599	2010-10-22 07:00:00
+Baltimore	New York	2010-10-22	54.0	1142	2010-10-22 07:00:00
+Baltimore	New York	2010-10-23	-10.0	1805	2010-10-23 07:00:00
+Baltimore	New York	2010-10-23	18.0	272	2010-10-23 07:00:00
+Baltimore	New York	2010-10-23	6.0	3171	2010-10-23 07:00:00
+Baltimore	New York	2010-10-24	12.0	1599	2010-10-24 07:00:00
+Baltimore	New York	2010-10-24	20.0	2571	2010-10-24 07:00:00
+Baltimore	New York	2010-10-25	-25.0	1064	2010-10-25 07:00:00
+Baltimore	New York	2010-10-25	106.0	1599	2010-10-25 07:00:00
+Baltimore	New York	2010-10-25	92.0	1142	2010-10-25 07:00:00
 Baltimore	New York	2010-10-26	-22.0	1064	2010-10-26 07:00:00
 Baltimore	New York	2010-10-26	123.0	1142	2010-10-26 07:00:00
 Baltimore	New York	2010-10-26	90.0	1599	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	12.0	361	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	0.0	897	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	29.0	1531	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	-17.0	1610	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	6.0	3198	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	4.0	2630	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	-27.0	2646	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	-11.0	2662	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	13.0	3014	2010-10-26 07:00:00
-Washington	New York	2010-10-26	4.0	7291	2010-10-26 07:00:00
-Baltimore	New York	2010-10-21	17.0	1064	2010-10-21 07:00:00
-Baltimore	New York	2010-10-21	105.0	1142	2010-10-21 07:00:00
-Baltimore	New York	2010-10-21	28.0	1599	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	142.0	361	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	77.0	897	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	53.0	1531	2010-10-21 07:00:00
+Baltimore	New York	2010-10-27	-18.0	1064	2010-10-27 07:00:00
+Baltimore	New York	2010-10-27	49.0	1142	2010-10-27 07:00:00
+Baltimore	New York	2010-10-27	92.0	1599	2010-10-27 07:00:00
+Baltimore	New York	2010-10-28	-14.0	1142	2010-10-28 07:00:00
+Baltimore	New York	2010-10-28	-14.0	1599	2010-10-28 07:00:00
+Baltimore	New York	2010-10-28	-4.0	1064	2010-10-28 07:00:00
+Baltimore	New York	2010-10-29	-2.0	1599	2010-10-29 07:00:00
+Baltimore	New York	2010-10-29	-24.0	1064	2010-10-29 07:00:00
+Baltimore	New York	2010-10-29	21.0	1142	2010-10-29 07:00:00
+Baltimore	New York	2010-10-30	-1.0	1805	2010-10-30 07:00:00
+Baltimore	New York	2010-10-30	14.0	272	2010-10-30 07:00:00
+Baltimore	New York	2010-10-30	5.0	3171	2010-10-30 07:00:00
+Baltimore	New York	2010-10-31	-1.0	1599	2010-10-31 07:00:00
+Baltimore	New York	2010-10-31	-14.0	2571	2010-10-31 07:00:00
+Chicago	New York	2010-10-20	-2.0	3198	2010-10-20 07:00:00
+Chicago	New York	2010-10-20	-6.0	1610	2010-10-20 07:00:00
+Chicago	New York	2010-10-20	15.0	1531	2010-10-20 07:00:00
+Chicago	New York	2010-10-20	24.0	897	2010-10-20 07:00:00
+Chicago	New York	2010-10-20	42.0	361	2010-10-20 07:00:00
 Chicago	New York	2010-10-21	-5.0	1610	2010-10-21 07:00:00
+Chicago	New York	2010-10-21	142.0	361	2010-10-21 07:00:00
 Chicago	New York	2010-10-21	51.0	3198	2010-10-21 07:00:00
-Cleveland	New York	2010-10-21	3.0	2630	2010-10-21 07:00:00
-Cleveland	New York	2010-10-21	29.0	2646	2010-10-21 07:00:00
-Cleveland	New York	2010-10-21	72.0	3014	2010-10-21 07:00:00
-Washington	New York	2010-10-21	22.0	7291	2010-10-21 07:00:00
-Baltimore	New York	2010-10-25	-25.0	1064	2010-10-25 07:00:00
-Baltimore	New York	2010-10-25	92.0	1142	2010-10-25 07:00:00
-Baltimore	New York	2010-10-25	106.0	1599	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	31.0	361	2010-10-25 07:00:00
+Chicago	New York	2010-10-21	53.0	1531	2010-10-21 07:00:00
+Chicago	New York	2010-10-21	77.0	897	2010-10-21 07:00:00
+Chicago	New York	2010-10-22	-11.0	3198	2010-10-22 07:00:00
+Chicago	New York	2010-10-22	-6.0	1610	2010-10-22 07:00:00
+Chicago	New York	2010-10-22	16.0	1531	2010-10-22 07:00:00
+Chicago	New York	2010-10-22	2.0	361	2010-10-22 07:00:00
+Chicago	New York	2010-10-22	24.0	897	2010-10-22 07:00:00
+Chicago	New York	2010-10-23	1.0	650	2010-10-23 07:00:00
+Chicago	New York	2010-10-23	11.0	3085	2010-10-23 07:00:00
+Chicago	New York	2010-10-23	3.0	384	2010-10-23 07:00:00
+Chicago	New York	2010-10-23	32.0	426	2010-10-23 07:00:00
+Chicago	New York	2010-10-24	-17.0	1610	2010-10-24 07:00:00
+Chicago	New York	2010-10-24	-3.0	3198	2010-10-24 07:00:00
+Chicago	New York	2010-10-24	-5.0	1531	2010-10-24 07:00:00
+Chicago	New York	2010-10-24	10.0	361	2010-10-24 07:00:00
+Chicago	New York	2010-10-24	113.0	897	2010-10-24 07:00:00
 Chicago	New York	2010-10-25	-1.0	897	2010-10-25 07:00:00
+Chicago	New York	2010-10-25	-16.0	3198	2010-10-25 07:00:00
+Chicago	New York	2010-10-25	31.0	361	2010-10-25 07:00:00
 Chicago	New York	2010-10-25	43.0	1531	2010-10-25 07:00:00
 Chicago	New York	2010-10-25	6.0	1610	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	-16.0	3198	2010-10-25 07:00:00
-Cleveland	New York	2010-10-25	-4.0	2630	2010-10-25 07:00:00
-Cleveland	New York	2010-10-25	81.0	2646	2010-10-25 07:00:00
-Cleveland	New York	2010-10-25	42.0	3014	2010-10-25 07:00:00
-Washington	New York	2010-10-25	9.0	7291	2010-10-25 07:00:00
-Baltimore	New York	2010-10-24	12.0	1599	2010-10-24 07:00:00
-Baltimore	New York	2010-10-24	20.0	2571	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	10.0	361	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	113.0	897	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	-5.0	1531	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	-17.0	1610	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	-3.0	3198	2010-10-24 07:00:00
-Cleveland	New York	2010-10-24	5.0	2254	2010-10-24 07:00:00
-Cleveland	New York	2010-10-24	-11.0	2630	2010-10-24 07:00:00
-Cleveland	New York	2010-10-24	-20.0	2646	2010-10-24 07:00:00
-Cleveland	New York	2010-10-24	-9.0	3014	2010-10-24 07:00:00
-Washington	New York	2010-10-24	-26.0	7282	2010-10-24 07:00:00
-Baltimore	New York	2010-10-23	18.0	272	2010-10-23 07:00:00
-Baltimore	New York	2010-10-23	-10.0	1805	2010-10-23 07:00:00
-Baltimore	New York	2010-10-23	6.0	3171	2010-10-23 07:00:00
-Chicago	New York	2010-10-23	3.0	384	2010-10-23 07:00:00
-Chicago	New York	2010-10-23	32.0	426	2010-10-23 07:00:00
-Chicago	New York	2010-10-23	1.0	650	2010-10-23 07:00:00
-Chicago	New York	2010-10-23	11.0	3085	2010-10-23 07:00:00
-Cleveland	New York	2010-10-23	-21.0	2932	2010-10-23 07:00:00
-Washington	New York	2010-10-23	-25.0	5832	2010-10-23 07:00:00
-Washington	New York	2010-10-23	-21.0	5904	2010-10-23 07:00:00
-Washington	New York	2010-10-23	-18.0	5917	2010-10-23 07:00:00
-Washington	New York	2010-10-23	-16.0	7274	2010-10-23 07:00:00
-Baltimore	New York	2010-10-29	-24.0	1064	2010-10-29 07:00:00
-Baltimore	New York	2010-10-29	21.0	1142	2010-10-29 07:00:00
-Baltimore	New York	2010-10-29	-2.0	1599	2010-10-29 07:00:00
-Chicago	New York	2010-10-29	-12.0	361	2010-10-29 07:00:00
-Chicago	New York	2010-10-29	-11.0	897	2010-10-29 07:00:00
-Chicago	New York	2010-10-29	15.0	1531	2010-10-29 07:00:00
-Chicago	New York	2010-10-29	-18.0	1610	2010-10-29 07:00:00
-Chicago	New York	2010-10-29	-4.0	3198	2010-10-29 07:00:00
-Cleveland	New York	2010-10-29	-4.0	2630	2010-10-29 07:00:00
-Cleveland	New York	2010-10-29	-19.0	2646	2010-10-29 07:00:00
-Cleveland	New York	2010-10-29	-12.0	3014	2010-10-29 07:00:00
-Washington	New York	2010-10-29	1.0	7291	2010-10-29 07:00:00
-Baltimore	New York	2010-10-27	-18.0	1064	2010-10-27 07:00:00
-Baltimore	New York	2010-10-27	49.0	1142	2010-10-27 07:00:00
-Baltimore	New York	2010-10-27	92.0	1599	2010-10-27 07:00:00
-Chicago	New York	2010-10-27	148.0	361	2010-10-27 07:00:00
+Chicago	New York	2010-10-26	-17.0	1610	2010-10-26 07:00:00
+Chicago	New York	2010-10-26	0.0	897	2010-10-26 07:00:00
+Chicago	New York	2010-10-26	12.0	361	2010-10-26 07:00:00
+Chicago	New York	2010-10-26	29.0	1531	2010-10-26 07:00:00
+Chicago	New York	2010-10-26	6.0	3198	2010-10-26 07:00:00
 Chicago	New York	2010-10-27	-11.0	897	2010-10-27 07:00:00
+Chicago	New York	2010-10-27	148.0	361	2010-10-27 07:00:00
+Chicago	New York	2010-10-27	21.0	3198	2010-10-27 07:00:00
 Chicago	New York	2010-10-27	70.0	1531	2010-10-27 07:00:00
 Chicago	New York	2010-10-27	8.0	1610	2010-10-27 07:00:00
-Chicago	New York	2010-10-27	21.0	3198	2010-10-27 07:00:00
-Cleveland	New York	2010-10-27	16.0	2630	2010-10-27 07:00:00
-Cleveland	New York	2010-10-27	27.0	3014	2010-10-27 07:00:00
-Washington	New York	2010-10-27	26.0	7291	2010-10-27 07:00:00
-Baltimore	New York	2010-10-28	-4.0	1064	2010-10-28 07:00:00
-Baltimore	New York	2010-10-28	-14.0	1142	2010-10-28 07:00:00
-Baltimore	New York	2010-10-28	-14.0	1599	2010-10-28 07:00:00
+Chicago	New York	2010-10-28	-11.0	1531	2010-10-28 07:00:00
+Chicago	New York	2010-10-28	-18.0	3198	2010-10-28 07:00:00
 Chicago	New York	2010-10-28	2.0	361	2010-10-28 07:00:00
 Chicago	New York	2010-10-28	2.0	897	2010-10-28 07:00:00
-Chicago	New York	2010-10-28	-11.0	1531	2010-10-28 07:00:00
 Chicago	New York	2010-10-28	3.0	1610	2010-10-28 07:00:00
-Chicago	New York	2010-10-28	-18.0	3198	2010-10-28 07:00:00
-Cleveland	New York	2010-10-28	3.0	2630	2010-10-28 07:00:00
-Cleveland	New York	2010-10-28	-6.0	2646	2010-10-28 07:00:00
-Cleveland	New York	2010-10-28	1.0	3014	2010-10-28 07:00:00
-Washington	New York	2010-10-28	45.0	7291	2010-10-28 07:00:00
-Baltimore	New York	2010-10-30	14.0	272	2010-10-30 07:00:00
-Baltimore	New York	2010-10-30	-1.0	1805	2010-10-30 07:00:00
-Baltimore	New York	2010-10-30	5.0	3171	2010-10-30 07:00:00
-Chicago	New York	2010-10-30	-6.0	384	2010-10-30 07:00:00
+Chicago	New York	2010-10-29	-11.0	897	2010-10-29 07:00:00
+Chicago	New York	2010-10-29	-12.0	361	2010-10-29 07:00:00
+Chicago	New York	2010-10-29	-18.0	1610	2010-10-29 07:00:00
+Chicago	New York	2010-10-29	-4.0	3198	2010-10-29 07:00:00
+Chicago	New York	2010-10-29	15.0	1531	2010-10-29 07:00:00
 Chicago	New York	2010-10-30	-10.0	426	2010-10-30 07:00:00
-Chicago	New York	2010-10-30	-5.0	650	2010-10-30 07:00:00
 Chicago	New York	2010-10-30	-5.0	3085	2010-10-30 07:00:00
-Cleveland	New York	2010-10-30	-23.0	2018	2010-10-30 07:00:00
-Cleveland	New York	2010-10-30	-12.0	2932	2010-10-30 07:00:00
-Washington	New York	2010-10-30	-27.0	5904	2010-10-30 07:00:00
-Washington	New York	2010-10-30	-16.0	5917	2010-10-30 07:00:00
-Baltimore	New York	2010-10-20	-30.0	1064	2010-10-20 07:00:00
-Baltimore	New York	2010-10-20	23.0	1142	2010-10-20 07:00:00
-Baltimore	New York	2010-10-20	6.0	1599	2010-10-20 07:00:00
-Chicago	New York	2010-10-20	42.0	361	2010-10-20 07:00:00
-Chicago	New York	2010-10-20	24.0	897	2010-10-20 07:00:00
-Chicago	New York	2010-10-20	15.0	1531	2010-10-20 07:00:00
-Chicago	New York	2010-10-20	-6.0	1610	2010-10-20 07:00:00
-Chicago	New York	2010-10-20	-2.0	3198	2010-10-20 07:00:00
-Cleveland	New York	2010-10-20	-8.0	2630	2010-10-20 07:00:00
+Chicago	New York	2010-10-30	-5.0	650	2010-10-30 07:00:00
+Chicago	New York	2010-10-30	-6.0	384	2010-10-30 07:00:00
+Chicago	New York	2010-10-31	-15.0	3198	2010-10-31 07:00:00
+Chicago	New York	2010-10-31	-18.0	897	2010-10-31 07:00:00
+Chicago	New York	2010-10-31	-22.0	1610	2010-10-31 07:00:00
+Chicago	New York	2010-10-31	-25.0	361	2010-10-31 07:00:00
+Chicago	New York	2010-10-31	-4.0	1531	2010-10-31 07:00:00
 Cleveland	New York	2010-10-20	-15.0	3014	2010-10-20 07:00:00
-Washington	New York	2010-10-20	-2.0	7291	2010-10-20 07:00:00
-Baltimore	New York	2010-10-22	-12.0	1064	2010-10-22 07:00:00
-Baltimore	New York	2010-10-22	54.0	1142	2010-10-22 07:00:00
-Baltimore	New York	2010-10-22	18.0	1599	2010-10-22 07:00:00
-Chicago	New York	2010-10-22	2.0	361	2010-10-22 07:00:00
-Chicago	New York	2010-10-22	24.0	897	2010-10-22 07:00:00
-Chicago	New York	2010-10-22	16.0	1531	2010-10-22 07:00:00
-Chicago	New York	2010-10-22	-6.0	1610	2010-10-22 07:00:00
-Chicago	New York	2010-10-22	-11.0	3198	2010-10-22 07:00:00
-Cleveland	New York	2010-10-22	1.0	2630	2010-10-22 07:00:00
+Cleveland	New York	2010-10-20	-8.0	2630	2010-10-20 07:00:00
+Cleveland	New York	2010-10-21	29.0	2646	2010-10-21 07:00:00
+Cleveland	New York	2010-10-21	3.0	2630	2010-10-21 07:00:00
+Cleveland	New York	2010-10-21	72.0	3014	2010-10-21 07:00:00
 Cleveland	New York	2010-10-22	-25.0	2646	2010-10-22 07:00:00
 Cleveland	New York	2010-10-22	-3.0	3014	2010-10-22 07:00:00
-Baltimore	New York	2010-10-31	-1.0	1599	2010-10-31 07:00:00
-Baltimore	New York	2010-10-31	-14.0	2571	2010-10-31 07:00:00
-Chicago	New York	2010-10-31	-25.0	361	2010-10-31 07:00:00
-Chicago	New York	2010-10-31	-18.0	897	2010-10-31 07:00:00
-Chicago	New York	2010-10-31	-4.0	1531	2010-10-31 07:00:00
-Chicago	New York	2010-10-31	-22.0	1610	2010-10-31 07:00:00
-Chicago	New York	2010-10-31	-15.0	3198	2010-10-31 07:00:00
+Cleveland	New York	2010-10-22	1.0	2630	2010-10-22 07:00:00
+Cleveland	New York	2010-10-23	-21.0	2932	2010-10-23 07:00:00
+Cleveland	New York	2010-10-24	-11.0	2630	2010-10-24 07:00:00
+Cleveland	New York	2010-10-24	-20.0	2646	2010-10-24 07:00:00
+Cleveland	New York	2010-10-24	-9.0	3014	2010-10-24 07:00:00
+Cleveland	New York	2010-10-24	5.0	2254	2010-10-24 07:00:00
+Cleveland	New York	2010-10-25	-4.0	2630	2010-10-25 07:00:00
+Cleveland	New York	2010-10-25	42.0	3014	2010-10-25 07:00:00
+Cleveland	New York	2010-10-25	81.0	2646	2010-10-25 07:00:00
+Cleveland	New York	2010-10-26	-11.0	2662	2010-10-26 07:00:00
+Cleveland	New York	2010-10-26	-27.0	2646	2010-10-26 07:00:00
+Cleveland	New York	2010-10-26	13.0	3014	2010-10-26 07:00:00
+Cleveland	New York	2010-10-26	4.0	2630	2010-10-26 07:00:00
+Cleveland	New York	2010-10-27	16.0	2630	2010-10-27 07:00:00
+Cleveland	New York	2010-10-27	27.0	3014	2010-10-27 07:00:00
+Cleveland	New York	2010-10-28	-6.0	2646	2010-10-28 07:00:00
+Cleveland	New York	2010-10-28	1.0	3014	2010-10-28 07:00:00
+Cleveland	New York	2010-10-28	3.0	2630	2010-10-28 07:00:00
+Cleveland	New York	2010-10-29	-12.0	3014	2010-10-29 07:00:00
+Cleveland	New York	2010-10-29	-19.0	2646	2010-10-29 07:00:00
+Cleveland	New York	2010-10-29	-4.0	2630	2010-10-29 07:00:00
+Cleveland	New York	2010-10-30	-12.0	2932	2010-10-30 07:00:00
+Cleveland	New York	2010-10-30	-23.0	2018	2010-10-30 07:00:00
+Washington	New York	2010-10-20	-2.0	7291	2010-10-20 07:00:00
+Washington	New York	2010-10-21	22.0	7291	2010-10-21 07:00:00
+Washington	New York	2010-10-23	-16.0	7274	2010-10-23 07:00:00
+Washington	New York	2010-10-23	-18.0	5917	2010-10-23 07:00:00
+Washington	New York	2010-10-23	-21.0	5904	2010-10-23 07:00:00
+Washington	New York	2010-10-23	-25.0	5832	2010-10-23 07:00:00
+Washington	New York	2010-10-24	-26.0	7282	2010-10-24 07:00:00
+Washington	New York	2010-10-25	9.0	7291	2010-10-25 07:00:00
+Washington	New York	2010-10-26	4.0	7291	2010-10-26 07:00:00
+Washington	New York	2010-10-27	26.0	7291	2010-10-27 07:00:00
+Washington	New York	2010-10-28	45.0	7291	2010-10-28 07:00:00
+Washington	New York	2010-10-29	1.0	7291	2010-10-29 07:00:00
+Washington	New York	2010-10-30	-16.0	5917	2010-10-30 07:00:00
+Washington	New York	2010-10-30	-27.0	5904	2010-10-30 07:00:00
 Washington	New York	2010-10-31	-18.0	7282	2010-10-31 07:00:00
 PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25
 PREHOOK: type: QUERY
@@ -1900,29 +1900,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-
 #### A masked pattern was here ####
 Baltimore	New York	2010-10-23	18.0	272	2010-10-23 07:00:00
 Baltimore	New York	2010-10-30	14.0	272	2010-10-30 07:00:00
+Chicago	New York	2010-10-20	24.0	897	2010-10-20 07:00:00
 Chicago	New York	2010-10-20	42.0	361	2010-10-20 07:00:00
 Chicago	New York	2010-10-21	142.0	361	2010-10-21 07:00:00
+Chicago	New York	2010-10-21	77.0	897	2010-10-21 07:00:00
 Chicago	New York	2010-10-22	2.0	361	2010-10-22 07:00:00
+Chicago	New York	2010-10-22	24.0	897	2010-10-22 07:00:00
+Chicago	New York	2010-10-23	1.0	650	2010-10-23 07:00:00
+Chicago	New York	2010-10-23	3.0	384	2010-10-23 07:00:00
+Chicago	New York	2010-10-23	32.0	426	2010-10-23 07:00:00
 Chicago	New York	2010-10-24	10.0	361	2010-10-24 07:00:00
+Chicago	New York	2010-10-24	113.0	897	2010-10-24 07:00:00
+Chicago	New York	2010-10-25	-1.0	897	2010-10-25 07:00:00
 Chicago	New York	2010-10-25	31.0	361	2010-10-25 07:00:00
+Chicago	New York	2010-10-26	0.0	897	2010-10-26 07:00:00
 Chicago	New York	2010-10-26	12.0	361	2010-10-26 07:00:00
+Chicago	New York	2010-10-27	-11.0	897	2010-10-27 07:00:00
 Chicago	New York	2010-10-27	148.0	361	2010-10-27 07:00:00
 Chicago	New York	2010-10-28	2.0	361	2010-10-28 07:00:00
 Chicago	New York	2010-10-29	-12.0	361	2010-10-29 07:00:00
-Chicago	New York	2010-10-31	-25.0	361	2010-10-31 07:00:00
-Chicago	New York	2010-10-23	3.0	384	2010-10-23 07:00:00
-Chicago	New York	2010-10-30	-6.0	384	2010-10-30 07:00:00
-Chicago	New York	2010-10-23	32.0	426	2010-10-23 07:00:00
 Chicago	New York	2010-10-30	-10.0	426	2010-10-30 07:00:00
-Chicago	New York	2010-10-23	1.0	650	2010-10-23 07:00:00
 Chicago	New York	2010-10-30	-5.0	650	2010-10-30 07:00:00
-Chicago	New York	2010-10-20	24.0	897	2010-10-20 07:00:00
-Chicago	New York	2010-10-21	77.0	897	2010-10-21 07:00:00
-Chicago	New York	2010-10-22	24.0	897	2010-10-22 07:00:00
-Chicago	New York	2010-10-24	113.0	897	2010-10-24 07:00:00
-Chicago	New York	2010-10-25	-1.0	897	2010-10-25 07:00:00
-Chicago	New York	2010-10-26	0.0	897	2010-10-26 07:00:00
-Chicago	New York	2010-10-27	-11.0	897	2010-10-27 07:00:00
+Chicago	New York	2010-10-30	-6.0	384	2010-10-30 07:00:00
+Chicago	New York	2010-10-31	-25.0	361	2010-10-31 07:00:00
 PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time
 PREHOOK: type: QUERY
 PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp
@@ -1955,17 +1955,17 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00
 #### A masked pattern was here ####
-2010-10-22 07:00:00	11
-2010-10-23 07:00:00	12
-2010-10-30 07:00:00	11
 2010-10-20 07:00:00	11
 2010-10-21 07:00:00	12
+2010-10-22 07:00:00	11
+2010-10-23 07:00:00	12
 2010-10-24 07:00:00	12
 2010-10-25 07:00:00	12
 2010-10-26 07:00:00	13
 2010-10-27 07:00:00	11
 2010-10-28 07:00:00	12
 2010-10-29 07:00:00	12
+2010-10-30 07:00:00	11
 2010-10-31 07:00:00	8
 PREHOOK: query: explain vectorization expression
 select * from flights_tiny_orc_partitioned_timestamp
@@ -2018,7 +2018,7 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: flights_tiny_orc_partitioned_timestamp
-                  Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 137 Data size: 96472 Basic stats: COMPLETE Column stats: PARTIAL
                   TableScan Vectorization:
                       native: true
                   Select Operator
@@ -2028,13 +2028,13 @@ STAGE PLANS:
                         className: VectorSelectOperator
                         native: true
                         projectedOutputColumnNums: [0, 1, 2, 3, 4, 5]
-                    Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL
                     File Output Operator
                       compressed: false
                       File Sink Vectorization:
                           className: VectorFileSinkOperator
                           native: false
-                      Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL
                       table:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -2077,154 +2077,154 @@ POSTHOOK: query: select * from flights_tiny_orc_partitioned_timestamp
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp
 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-20 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-21 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-22 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00
-POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00
-#### A masked pattern was here ####
-Baltimore	New York	2010-10-26	-22.0	1064	2010-10-26 07:00:00
-Baltimore	New York	2010-10-26	123.0	1142	2010-10-26 07:00:00
-Baltimore	New York	2010-10-26	90.0	1599	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	12.0	361	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	0.0	897	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	29.0	1531	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	-17.0	1610	2010-10-26 07:00:00
-Chicago	New York	2010-10-26	6.0	3198	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	4.0	2630	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	-27.0	2646	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	-11.0	2662	2010-10-26 07:00:00
-Cleveland	New York	2010-10-26	13.0	3014	2010-10-26 07:00:00
-Washington	New York	2010-10-26	4.0	7291	2010-10-26 07:00:00
-Baltimore	New York	2010-10-21	17.0	1064	2010-10-21 07:00:00
-Baltimore	New York	2010-10-21	105.0	1142	2010-10-21 07:00:00
-Baltimore	New York	2010-10-21	28.0	1599	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	142.0	361	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	77.0	897	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	53.0	1531	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	-5.0	1610	2010-10-21 07:00:00
-Chicago	New York	2010-10-21	51.0	3198	2010-10-21 07:00:00
-Cleveland	New York	2010-10-21	3.0	2630	2010-10-21 07:00:00
-Cleveland	New York	2010-10-21	29.0	2646	2010-10-21 07:00:00
-Cleveland	New York	2010-10-21	72.0	3014	2010-10-21 07:00:00
-Washington	New York	2010-10-21	22.0	7291	2010-10-21 07:00:00
-Baltimore	New York	2010-10-25	-25.0	1064	2010-10-25 07:00:00
-Baltimore	New York	2010-10-25	92.0	1142	2010-10-25 07:00:00
-Baltimore	New York	2010-10-25	106.0	1599	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	31.0	361	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	-1.0	897	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	43.0	1531	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	6.0	1610	2010-10-25 07:00:00
-Chicago	New York	2010-10-25	-16.0	3198	2010-10-25 07:00:00
-Cleveland	New York	2010-10-25	-4.0	2630	2010-10-25 07:00:00
-Cleveland	New York	2010-10-25	81.0	2646	2010-10-25 07:00:00
-Cleveland	New York	2010-10-25	42.0	3014	2010-10-25 07:00:00
-Washington	New York	2010-10-25	9.0	7291	2010-10-25 07:00:00
-Baltimore	New York	2010-10-24	12.0	1599	2010-10-24 07:00:00
-Baltimore	New York	2010-10-24	20.0	2571	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	10.0	361	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	113.0	897	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	-5.0	1531	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	-17.0	1610	2010-10-24 07:00:00
-Chicago	New York	2010-10-24	-3.0	3198	2010-10-24 07:00:00
-Cleveland	New York	2010-10-24	5.0	2254	2010-10-24 07:00:00
-Cleveland	Ne

<TRUNCATED>

[3/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out
index c024062..9255be7 100644
--- a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out
+++ b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out
@@ -52,52 +52,28 @@ STAGE PLANS:
                     expressions: key (type: string), value (type: string)
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          name: default.nzhang_part5
-                    Select Operator
-                      expressions: _col0 (type: string), _col1 (type: string)
-                      outputColumnNames: key, value
-                      Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll')
-                        keys: value (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string)
-                          sort order: +
-                          Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
-            Execution mode: llap
+                    Reduce Output Operator
+                      key expressions: _col1 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col1 (type: string)
+                      value expressions: _col0 (type: string)
+            Execution mode: vectorized, llap
             LLAP IO: no inputs
         Reducer 2 
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0)
-                keys: KEY._col0 (type: string)
-                mode: mergepartial
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY._col1 (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col1 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.nzhang_part5
 
   Stage: Stage-2
     Dependency Collection

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
index 2c838b6..4ed30ee 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
@@ -60,7 +60,6 @@ STAGE PLANS:
                         key expressions: _col1 (type: string)
                         sort order: +
                         Map-reduce partition columns: _col1 (type: string)
-                        Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: may be used (ACID table)
@@ -70,7 +69,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: string), KEY._col1 (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/mm_dp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_dp.q.out b/ql/src/test/results/clientpositive/llap/mm_dp.q.out
index b23a3d7..6ad05e9 100644
--- a/ql/src/test/results/clientpositive/llap/mm_dp.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_dp.q.out
@@ -149,7 +149,6 @@ STAGE PLANS:
                       key expressions: _col3 (type: string), _bucket_number (type: string), _col2 (type: int)
                       sort order: +++
                       Map-reduce partition columns: _col3 (type: string)
-                      Statistics: Num rows: 2605 Data size: 268315 Basic stats: COMPLETE Column stats: COMPLETE
                       value expressions: _col0 (type: int), _col1 (type: int)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -159,11 +158,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY._col2 (type: int), KEY._col3 (type: string), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number
-                Statistics: Num rows: 2605 Data size: 747635 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 2605 Data size: 747635 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2605 Data size: 268315 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_analyze.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out
index 25426bf..1389de0 100644
--- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out
@@ -772,7 +772,7 @@ Partition Parameters:
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22043               
-	totalSize           	5339                
+	totalSize           	5336                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -866,7 +866,7 @@ Partition Parameters:
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22043               
-	totalSize           	5339                
+	totalSize           	5336                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -1009,7 +1009,7 @@ Partition Parameters:
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22043               
-	totalSize           	5339                
+	totalSize           	5336                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_merge1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out
index d4f9517..5bb6432 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out
@@ -76,52 +76,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1_n1
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1_n1
 
   Stage: Stage-2
     Dependency Collection
@@ -164,13 +140,8 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).key EXPRESSION [(src
 POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-Found 6 items
--rw-r--r--   3 ### USER ### ### GROUP ###        555 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        562 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        561 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        496 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        554 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        478 ### HDFS DATE ### hdfs://### HDFS PATH ###
+Found 1 items
+-rw-rw-rw-   3 ### USER ### ### GROUP ###       1359 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: EXPLAIN
     INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part)
         SELECT key, value, PMOD(HASH(key), 2) as part
@@ -212,52 +183,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1b_n1
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1b_n1
 
   Stage: Stage-8
     Conditional Operator
@@ -346,7 +293,7 @@ POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)
 POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 Found 1 items
--rw-rw-rw-   3 ### USER ### ### GROUP ###       1360 ### HDFS DATE ### hdfs://### HDFS PATH ###
+-rw-rw-rw-   3 ### USER ### ### GROUP ###       1359 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: EXPLAIN
     INSERT OVERWRITE TABLE orcfile_merge1c_n1 PARTITION (ds='1', part)
         SELECT key, value, PMOD(HASH(key), 2) as part
@@ -388,52 +335,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1c_n1
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1c_n1
 
   Stage: Stage-8
     Conditional Operator
@@ -514,7 +437,7 @@ POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)
 POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 Found 1 items
--rw-rw-rw-   3 ### USER ### ### GROUP ###       2461 ### HDFS DATE ### hdfs://### HDFS PATH ###
+-rw-rw-rw-   3 ### USER ### ### GROUP ###       1359 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: SELECT SUM(HASH(c)) FROM (
     SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
     FROM orcfile_merge1_n1 WHERE ds='1'

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
index 95b352a..d943b40 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
@@ -76,52 +76,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1
 
   Stage: Stage-2
     Dependency Collection
@@ -164,10 +140,8 @@ POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)sr
 POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-Found 3 items
--rw-r--r--   3 ### USER ### ### GROUP ###        947 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        875 ### HDFS DATE ### hdfs://### HDFS PATH ###
--rw-r--r--   3 ### USER ### ### GROUP ###        856 ### HDFS DATE ### hdfs://### HDFS PATH ###
+Found 1 items
+-rw-rw-rw-   3 ### USER ### ### GROUP ###       1754 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: EXPLAIN
     INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part)
         SELECT key, value, PMOD(HASH(key), 2) as part
@@ -209,52 +183,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1b
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1b
 
   Stage: Stage-8
     Conditional Operator
@@ -385,52 +335,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1c
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1c
 
   Stage: Stage-8
     Conditional Operator
@@ -511,7 +437,7 @@ POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src
 POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 Found 1 items
--rw-rw-rw-   3 ### USER ### ### GROUP ###       2409 ### HDFS DATE ### hdfs://### HDFS PATH ###
+-rw-rw-rw-   3 ### USER ### ### GROUP ###       1754 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: SELECT SUM(HASH(c)) FROM (
     SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
     FROM orcfile_merge1 WHERE ds='1'
@@ -618,7 +544,7 @@ POSTHOOK: type: ALTER_PARTITION_MERGE
 POSTHOOK: Input: default@orcfile_merge1
 POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0
 Found 1 items
--rw-rw-rw-   3 ### USER ### ### GROUP ###       2409 ### HDFS DATE ### hdfs://### HDFS PATH ###
+-rw-rw-rw-   3 ### USER ### ### GROUP ###       1754 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: SELECT SUM(HASH(c)) FROM (
     SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
     FROM orcfile_merge1c WHERE ds='1'
@@ -698,80 +624,41 @@ Type: struct<key:int,value:string>
 
 Stripe Statistics:
   Stripe 1:
-    Column 0: count: 90 hasNull: false
-    Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736
-    Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612
-  Stripe 2:
-    Column 0: count: 78 hasNull: false
-    Column 1: count: 78 hasNull: false bytesOnDisk: 161 min: 0 max: 497 sum: 18371
-    Column 2: count: 78 hasNull: false bytesOnDisk: 380 min: val_0 max: val_95 sum: 529
-  Stripe 3:
-    Column 0: count: 74 hasNull: false
-    Column 1: count: 74 hasNull: false bytesOnDisk: 153 min: 2 max: 493 sum: 19663
-    Column 2: count: 74 hasNull: false bytesOnDisk: 363 min: val_105 max: val_97 sum: 505
+    Column 0: count: 242 hasNull: false
+    Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770
+    Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646
 
 File Statistics:
   Column 0: count: 242 hasNull: false
-  Column 1: count: 242 hasNull: false bytesOnDisk: 499 min: 0 max: 497 sum: 60770
-  Column 2: count: 242 hasNull: false bytesOnDisk: 1171 min: val_0 max: val_97 sum: 1646
+  Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770
+  Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646
 
 Stripes:
-  Stripe: offset: 3 data: 613 rows: 90 tail: 61 index: 76
-    Stream: column 0 section ROW_INDEX start: 3 length 11
-    Stream: column 1 section ROW_INDEX start: 14 length 27
-    Stream: column 2 section ROW_INDEX start: 41 length 38
-    Stream: column 1 section DATA start: 79 length 185
-    Stream: column 2 section DATA start: 264 length 377
-    Stream: column 2 section LENGTH start: 641 length 51
+  Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77
+    Stream: column 0 section ROW_INDEX start: 3 length 12
+    Stream: column 1 section ROW_INDEX start: 15 length 28
+    Stream: column 2 section ROW_INDEX start: 43 length 37
+    Stream: column 1 section DATA start: 80 length 489
+    Stream: column 2 section DATA start: 569 length 247
+    Stream: column 2 section LENGTH start: 816 length 71
+    Stream: column 2 section DICTIONARY_DATA start: 887 length 592
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
-    Encoding column 2: DIRECT_V2
+    Encoding column 2: DICTIONARY_V2[153]
     Row group indices for column 0:
-      Entry 0: count: 90 hasNull: false positions: 
+      Entry 0: count: 242 hasNull: false positions: 
     Row group indices for column 1:
-      Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0
+      Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0
     Row group indices for column 2:
-      Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0
-  Stripe: offset: 753 data: 541 rows: 78 tail: 61 index: 76
-    Stream: column 0 section ROW_INDEX start: 753 length 11
-    Stream: column 1 section ROW_INDEX start: 764 length 27
-    Stream: column 2 section ROW_INDEX start: 791 length 38
-    Stream: column 1 section DATA start: 829 length 161
-    Stream: column 2 section DATA start: 990 length 332
-    Stream: column 2 section LENGTH start: 1322 length 48
-    Encoding column 0: DIRECT
-    Encoding column 1: DIRECT_V2
-    Encoding column 2: DIRECT_V2
-    Row group indices for column 0:
-      Entry 0: count: 78 hasNull: false positions: 
-    Row group indices for column 1:
-      Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0
-    Row group indices for column 2:
-      Entry 0: count: 78 hasNull: false min: val_0 max: val_95 sum: 529 positions: 0,0,0,0,0
-  Stripe: offset: 1431 data: 516 rows: 74 tail: 61 index: 78
-    Stream: column 0 section ROW_INDEX start: 1431 length 11
-    Stream: column 1 section ROW_INDEX start: 1442 length 27
-    Stream: column 2 section ROW_INDEX start: 1469 length 40
-    Stream: column 1 section DATA start: 1509 length 153
-    Stream: column 2 section DATA start: 1662 length 331
-    Stream: column 2 section LENGTH start: 1993 length 32
-    Encoding column 0: DIRECT
-    Encoding column 1: DIRECT_V2
-    Encoding column 2: DIRECT_V2
-    Row group indices for column 0:
-      Entry 0: count: 74 hasNull: false positions: 
-    Row group indices for column 1:
-      Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0
-    Row group indices for column 2:
-      Entry 0: count: 74 hasNull: false min: val_105 max: val_97 sum: 505 positions: 0,0,0,0,0
+      Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0
 
-File length: 2409 bytes
+File length: 1754 bytes
 Padding length: 0 bytes
 Padding ratio: 0%
 ________________________________________________________________________________________________________________________
 
 -- END ORC FILE DUMP --
-86	val_86	1	0
+2	val_2	1	0
 PREHOOK: query: select * from orcfile_merge1c where ds='1' and part='0' limit 1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orcfile_merge1c
@@ -787,80 +674,41 @@ Type: struct<key:int,value:string>
 
 Stripe Statistics:
   Stripe 1:
-    Column 0: count: 90 hasNull: false
-    Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736
-    Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612
-  Stripe 2:
-    Column 0: count: 78 hasNull: false
-    Column 1: count: 78 hasNull: false bytesOnDisk: 161 min: 0 max: 497 sum: 18371
-    Column 2: count: 78 hasNull: false bytesOnDisk: 380 min: val_0 max: val_95 sum: 529
-  Stripe 3:
-    Column 0: count: 74 hasNull: false
-    Column 1: count: 74 hasNull: false bytesOnDisk: 153 min: 2 max: 493 sum: 19663
-    Column 2: count: 74 hasNull: false bytesOnDisk: 363 min: val_105 max: val_97 sum: 505
+    Column 0: count: 242 hasNull: false
+    Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770
+    Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646
 
 File Statistics:
   Column 0: count: 242 hasNull: false
-  Column 1: count: 242 hasNull: false bytesOnDisk: 499 min: 0 max: 497 sum: 60770
-  Column 2: count: 242 hasNull: false bytesOnDisk: 1171 min: val_0 max: val_97 sum: 1646
+  Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770
+  Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646
 
 Stripes:
-  Stripe: offset: 3 data: 613 rows: 90 tail: 61 index: 76
-    Stream: column 0 section ROW_INDEX start: 3 length 11
-    Stream: column 1 section ROW_INDEX start: 14 length 27
-    Stream: column 2 section ROW_INDEX start: 41 length 38
-    Stream: column 1 section DATA start: 79 length 185
-    Stream: column 2 section DATA start: 264 length 377
-    Stream: column 2 section LENGTH start: 641 length 51
-    Encoding column 0: DIRECT
-    Encoding column 1: DIRECT_V2
-    Encoding column 2: DIRECT_V2
-    Row group indices for column 0:
-      Entry 0: count: 90 hasNull: false positions: 
-    Row group indices for column 1:
-      Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0
-    Row group indices for column 2:
-      Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0
-  Stripe: offset: 753 data: 541 rows: 78 tail: 61 index: 76
-    Stream: column 0 section ROW_INDEX start: 753 length 11
-    Stream: column 1 section ROW_INDEX start: 764 length 27
-    Stream: column 2 section ROW_INDEX start: 791 length 38
-    Stream: column 1 section DATA start: 829 length 161
-    Stream: column 2 section DATA start: 990 length 332
-    Stream: column 2 section LENGTH start: 1322 length 48
-    Encoding column 0: DIRECT
-    Encoding column 1: DIRECT_V2
-    Encoding column 2: DIRECT_V2
-    Row group indices for column 0:
-      Entry 0: count: 78 hasNull: false positions: 
-    Row group indices for column 1:
-      Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0
-    Row group indices for column 2:
-      Entry 0: count: 78 hasNull: false min: val_0 max: val_95 sum: 529 positions: 0,0,0,0,0
-  Stripe: offset: 1431 data: 516 rows: 74 tail: 61 index: 78
-    Stream: column 0 section ROW_INDEX start: 1431 length 11
-    Stream: column 1 section ROW_INDEX start: 1442 length 27
-    Stream: column 2 section ROW_INDEX start: 1469 length 40
-    Stream: column 1 section DATA start: 1509 length 153
-    Stream: column 2 section DATA start: 1662 length 331
-    Stream: column 2 section LENGTH start: 1993 length 32
+  Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77
+    Stream: column 0 section ROW_INDEX start: 3 length 12
+    Stream: column 1 section ROW_INDEX start: 15 length 28
+    Stream: column 2 section ROW_INDEX start: 43 length 37
+    Stream: column 1 section DATA start: 80 length 489
+    Stream: column 2 section DATA start: 569 length 247
+    Stream: column 2 section LENGTH start: 816 length 71
+    Stream: column 2 section DICTIONARY_DATA start: 887 length 592
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
-    Encoding column 2: DIRECT_V2
+    Encoding column 2: DICTIONARY_V2[153]
     Row group indices for column 0:
-      Entry 0: count: 74 hasNull: false positions: 
+      Entry 0: count: 242 hasNull: false positions: 
     Row group indices for column 1:
-      Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0
+      Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0
     Row group indices for column 2:
-      Entry 0: count: 74 hasNull: false min: val_105 max: val_97 sum: 505 positions: 0,0,0,0,0
+      Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0
 
-File length: 2409 bytes
+File length: 1754 bytes
 Padding length: 0 bytes
 Padding ratio: 0%
 ________________________________________________________________________________________________________________________
 
 -- END ORC FILE DUMP --
-86	val_86	1	0
+2	val_2	1	0
 PREHOOK: query: DROP TABLE orcfile_merge1
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@orcfile_merge1

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_merge2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out
index 430ce16..7265292 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out
@@ -50,52 +50,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int)
                     outputColumnNames: _col0, _col1, _col2, _col3
                     Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge2a_n0
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string)
-                      outputColumnNames: key, value, one, two, three
-                      Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: one (type: string), two (type: string), three (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                        Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
-                          sort order: +++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
-                          Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col3 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col4 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int), _col3 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col2 (type: int), _col3 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col3 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col4 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string), _col2 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                  Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge2a_n0
 
   Stage: Stage-2
     Dependency Collection
@@ -196,7 +172,7 @@ POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=1).value SIMPL
 POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 Found 1 items
--rw-r--r--   3 ### USER ### ### GROUP ###        349 ### HDFS DATE ### hdfs://### HDFS PATH ###
+-rw-rw-rw-   3 ### USER ### ### GROUP ###        349 ### HDFS DATE ### hdfs://### HDFS PATH ###
 PREHOOK: query: SELECT SUM(HASH(c)) FROM (
     SELECT TRANSFORM(*) USING 'tr \t _' AS (c)
     FROM orcfile_merge2a_n0

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
index 1f67f7b..634f139 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
@@ -52,52 +52,28 @@ STAGE PLANS:
                     expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double)
                     outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
                     Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orc_merge5a_n0
-                    Select Operator
-                      expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double)
-                      outputColumnNames: userid, string1, subtype, decimal1, ts, st
-                      Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                      Group By Operator
-                        aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll')
-                        keys: st (type: double)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                        Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: double)
-                          sort order: +
-                          Map-reduce partition columns: _col0 (type: double)
-                          Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                          value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:binary>), _col4 (type: struct<columntype:string,min:decimal(38,0),max:decimal(38,0),countnulls:bigint,bitvector:binary>), _col5 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col5 (type: double)
+                      sort order: +
+                      Map-reduce partition columns: _col5 (type: double)
+                      value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp)
             Execution mode: llap
             LLAP IO: all inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4)
-                keys: KEY._col0 (type: double)
-                mode: mergepartial
+              Select Operator
+                expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col4 (type: struct<columntype:string,min:decimal(38,0),max:decimal(38,0),countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col5 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: double)
-                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orc_merge5a_n0
 
   Stage: Stage-2
     Dependency Collection
@@ -272,52 +248,28 @@ STAGE PLANS:
                     expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double)
                     outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
                     Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orc_merge5a_n0
-                    Select Operator
-                      expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double)
-                      outputColumnNames: userid, string1, subtype, decimal1, ts, st
-                      Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                      Group By Operator
-                        aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll')
-                        keys: st (type: double)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                        Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: double)
-                          sort order: +
-                          Map-reduce partition columns: _col0 (type: double)
-                          Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                          value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:binary>), _col4 (type: struct<columntype:string,min:decimal(38,0),max:decimal(38,0),countnulls:bigint,bitvector:binary>), _col5 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col5 (type: double)
+                      sort order: +
+                      Map-reduce partition columns: _col5 (type: double)
+                      value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp)
             Execution mode: llap
             LLAP IO: all inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4)
-                keys: KEY._col0 (type: double)
-                mode: mergepartial
+              Select Operator
+                expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col4 (type: struct<columntype:string,min:decimal(38,0),max:decimal(38,0),countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col5 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: double)
-                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orc_merge5a_n0
 
   Stage: Stage-8
     Conditional Operator


[4/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out
index 406c620..7ad7bfb 100644
--- a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out
+++ b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out
@@ -54,124 +54,124 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08
 #### A masked pattern was here ####
-165	val_165	2008-04-08
-484	val_484	2008-04-08
-150	val_150	2008-04-08
-224	val_224	2008-04-08
-66	val_66	2008-04-08
-213	val_213	2008-04-08
-374	val_374	2008-04-08
-495	val_495	2008-04-08
-37	val_37	2008-04-08
-327	val_327	2008-04-08
-15	val_15	2008-04-08
-338	val_338	2008-04-08
-459	val_459	2008-04-08
-466	val_466	2008-04-08
-396	val_396	2008-04-08
-309	val_309	2008-04-08
-367	val_367	2008-04-08
 0	val_0	2008-04-08
-455	val_455	2008-04-08
-316	val_316	2008-04-08
-345	val_345	2008-04-08
+0	val_0	2008-04-08
+0	val_0	2008-04-08
+103	val_103	2008-04-08
+103	val_103	2008-04-08
+11	val_11	2008-04-08
+114	val_114	2008-04-08
+118	val_118	2008-04-08
+118	val_118	2008-04-08
+125	val_125	2008-04-08
+125	val_125	2008-04-08
 129	val_129	2008-04-08
-378	val_378	2008-04-08
-4	val_4	2008-04-08
-356	val_356	2008-04-08
+129	val_129	2008-04-08
+136	val_136	2008-04-08
+143	val_143	2008-04-08
+15	val_15	2008-04-08
+15	val_15	2008-04-08
+150	val_150	2008-04-08
+158	val_158	2008-04-08
+165	val_165	2008-04-08
+165	val_165	2008-04-08
 169	val_169	2008-04-08
-125	val_125	2008-04-08
-437	val_437	2008-04-08
-286	val_286	2008-04-08
-187	val_187	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+172	val_172	2008-04-08
+172	val_172	2008-04-08
 176	val_176	2008-04-08
-459	val_459	2008-04-08
-51	val_51	2008-04-08
-103	val_103	2008-04-08
-239	val_239	2008-04-08
-213	val_213	2008-04-08
 176	val_176	2008-04-08
-275	val_275	2008-04-08
-260	val_260	2008-04-08
-404	val_404	2008-04-08
+183	val_183	2008-04-08
+187	val_187	2008-04-08
+187	val_187	2008-04-08
+187	val_187	2008-04-08
+19	val_19	2008-04-08
+190	val_190	2008-04-08
+194	val_194	2008-04-08
+202	val_202	2008-04-08
+213	val_213	2008-04-08
+213	val_213	2008-04-08
 217	val_217	2008-04-08
-84	val_84	2008-04-08
-466	val_466	2008-04-08
-8	val_8	2008-04-08
-411	val_411	2008-04-08
-172	val_172	2008-04-08
-129	val_129	2008-04-08
-158	val_158	2008-04-08
-0	val_0	2008-04-08
+217	val_217	2008-04-08
+224	val_224	2008-04-08
+224	val_224	2008-04-08
+228	val_228	2008-04-08
+235	val_235	2008-04-08
+239	val_239	2008-04-08
+239	val_239	2008-04-08
+242	val_242	2008-04-08
+242	val_242	2008-04-08
+257	val_257	2008-04-08
 26	val_26	2008-04-08
-165	val_165	2008-04-08
-327	val_327	2008-04-08
-51	val_51	2008-04-08
-404	val_404	2008-04-08
-95	val_95	2008-04-08
+26	val_26	2008-04-08
+260	val_260	2008-04-08
+275	val_275	2008-04-08
 282	val_282	2008-04-08
-187	val_187	2008-04-08
-316	val_316	2008-04-08
-169	val_169	2008-04-08
-77	val_77	2008-04-08
-0	val_0	2008-04-08
-118	val_118	2008-04-08
 282	val_282	2008-04-08
-419	val_419	2008-04-08
-15	val_15	2008-04-08
-118	val_118	2008-04-08
-19	val_19	2008-04-08
-224	val_224	2008-04-08
+286	val_286	2008-04-08
+305	val_305	2008-04-08
 309	val_309	2008-04-08
-389	val_389	2008-04-08
+309	val_309	2008-04-08
+316	val_316	2008-04-08
+316	val_316	2008-04-08
+316	val_316	2008-04-08
+323	val_323	2008-04-08
+327	val_327	2008-04-08
+327	val_327	2008-04-08
 327	val_327	2008-04-08
-242	val_242	2008-04-08
-392	val_392	2008-04-08
-242	val_242	2008-04-08
-396	val_396	2008-04-08
-95	val_95	2008-04-08
-11	val_11	2008-04-08
-143	val_143	2008-04-08
-228	val_228	2008-04-08
 33	val_33	2008-04-08
-103	val_103	2008-04-08
+338	val_338	2008-04-08
+341	val_341	2008-04-08
+345	val_345	2008-04-08
+356	val_356	2008-04-08
 367	val_367	2008-04-08
-239	val_239	2008-04-08
-480	val_480	2008-04-08
-202	val_202	2008-04-08
-316	val_316	2008-04-08
-235	val_235	2008-04-08
-80	val_80	2008-04-08
-44	val_44	2008-04-08
-466	val_466	2008-04-08
-257	val_257	2008-04-08
-190	val_190	2008-04-08
-114	val_114	2008-04-08
+367	val_367	2008-04-08
+37	val_37	2008-04-08
+37	val_37	2008-04-08
+374	val_374	2008-04-08
+378	val_378	2008-04-08
+389	val_389	2008-04-08
+392	val_392	2008-04-08
 396	val_396	2008-04-08
-217	val_217	2008-04-08
-125	val_125	2008-04-08
-187	val_187	2008-04-08
-480	val_480	2008-04-08
-491	val_491	2008-04-08
-305	val_305	2008-04-08
+396	val_396	2008-04-08
+396	val_396	2008-04-08
+4	val_4	2008-04-08
+400	val_400	2008-04-08
+404	val_404	2008-04-08
+404	val_404	2008-04-08
+411	val_411	2008-04-08
+419	val_419	2008-04-08
+437	val_437	2008-04-08
+44	val_44	2008-04-08
 444	val_444	2008-04-08
-169	val_169	2008-04-08
-323	val_323	2008-04-08
-480	val_480	2008-04-08
-136	val_136	2008-04-08
-172	val_172	2008-04-08
+448	val_448	2008-04-08
+455	val_455	2008-04-08
+459	val_459	2008-04-08
+459	val_459	2008-04-08
 462	val_462	2008-04-08
-26	val_26	2008-04-08
 462	val_462	2008-04-08
-341	val_341	2008-04-08
-183	val_183	2008-04-08
-84	val_84	2008-04-08
-37	val_37	2008-04-08
-448	val_448	2008-04-08
-194	val_194	2008-04-08
+466	val_466	2008-04-08
+466	val_466	2008-04-08
+466	val_466	2008-04-08
 477	val_477	2008-04-08
-169	val_169	2008-04-08
-400	val_400	2008-04-08
+480	val_480	2008-04-08
+480	val_480	2008-04-08
+480	val_480	2008-04-08
+484	val_484	2008-04-08
+491	val_491	2008-04-08
+495	val_495	2008-04-08
+51	val_51	2008-04-08
+51	val_51	2008-04-08
+66	val_66	2008-04-08
+77	val_77	2008-04-08
+8	val_8	2008-04-08
+80	val_80	2008-04-08
+84	val_84	2008-04-08
+84	val_84	2008-04-08
+95	val_95	2008-04-08
+95	val_95	2008-04-08
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -205,6 +205,9 @@ STAGE PLANS:
   Stage: Stage-1
     Tez
 #### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
       Vertices:
         Map 1 
             Map Operator Tree:
@@ -215,16 +218,28 @@ STAGE PLANS:
                     expressions: key (type: int), value (type: string), ds (type: string)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          name: default.srcbucket_mapjoin_n8
+                    Reduce Output Operator
+                      key expressions: _col2 (type: string)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: string)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: vectorized, llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.srcbucket_mapjoin_n8
 
   Stage: Stage-2
     Dependency Collection
@@ -265,124 +280,124 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08
 #### A masked pattern was here ####
-165	val_165	2008-04-08
-484	val_484	2008-04-08
-150	val_150	2008-04-08
-224	val_224	2008-04-08
-66	val_66	2008-04-08
-213	val_213	2008-04-08
-374	val_374	2008-04-08
-495	val_495	2008-04-08
-37	val_37	2008-04-08
-327	val_327	2008-04-08
-15	val_15	2008-04-08
-338	val_338	2008-04-08
-459	val_459	2008-04-08
-466	val_466	2008-04-08
-396	val_396	2008-04-08
-309	val_309	2008-04-08
-367	val_367	2008-04-08
 0	val_0	2008-04-08
-455	val_455	2008-04-08
-316	val_316	2008-04-08
-345	val_345	2008-04-08
-129	val_129	2008-04-08
-378	val_378	2008-04-08
-4	val_4	2008-04-08
-356	val_356	2008-04-08
-169	val_169	2008-04-08
-125	val_125	2008-04-08
-437	val_437	2008-04-08
-286	val_286	2008-04-08
-187	val_187	2008-04-08
-176	val_176	2008-04-08
-459	val_459	2008-04-08
-51	val_51	2008-04-08
+0	val_0	2008-04-08
+0	val_0	2008-04-08
 103	val_103	2008-04-08
-239	val_239	2008-04-08
-213	val_213	2008-04-08
-176	val_176	2008-04-08
-275	val_275	2008-04-08
-260	val_260	2008-04-08
-404	val_404	2008-04-08
-217	val_217	2008-04-08
-84	val_84	2008-04-08
-466	val_466	2008-04-08
-8	val_8	2008-04-08
-411	val_411	2008-04-08
-172	val_172	2008-04-08
+103	val_103	2008-04-08
+11	val_11	2008-04-08
+114	val_114	2008-04-08
+118	val_118	2008-04-08
+118	val_118	2008-04-08
+125	val_125	2008-04-08
+125	val_125	2008-04-08
+129	val_129	2008-04-08
 129	val_129	2008-04-08
+136	val_136	2008-04-08
+143	val_143	2008-04-08
+15	val_15	2008-04-08
+15	val_15	2008-04-08
+150	val_150	2008-04-08
 158	val_158	2008-04-08
-0	val_0	2008-04-08
-26	val_26	2008-04-08
 165	val_165	2008-04-08
-327	val_327	2008-04-08
-51	val_51	2008-04-08
-404	val_404	2008-04-08
-95	val_95	2008-04-08
-282	val_282	2008-04-08
-187	val_187	2008-04-08
-316	val_316	2008-04-08
+165	val_165	2008-04-08
 169	val_169	2008-04-08
-77	val_77	2008-04-08
-0	val_0	2008-04-08
-118	val_118	2008-04-08
-282	val_282	2008-04-08
-419	val_419	2008-04-08
-15	val_15	2008-04-08
-118	val_118	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+172	val_172	2008-04-08
+172	val_172	2008-04-08
+176	val_176	2008-04-08
+176	val_176	2008-04-08
+183	val_183	2008-04-08
+187	val_187	2008-04-08
+187	val_187	2008-04-08
+187	val_187	2008-04-08
 19	val_19	2008-04-08
+190	val_190	2008-04-08
+194	val_194	2008-04-08
+202	val_202	2008-04-08
+213	val_213	2008-04-08
+213	val_213	2008-04-08
+217	val_217	2008-04-08
+217	val_217	2008-04-08
+224	val_224	2008-04-08
 224	val_224	2008-04-08
-309	val_309	2008-04-08
-389	val_389	2008-04-08
-327	val_327	2008-04-08
-242	val_242	2008-04-08
-392	val_392	2008-04-08
-242	val_242	2008-04-08
-396	val_396	2008-04-08
-95	val_95	2008-04-08
-11	val_11	2008-04-08
-143	val_143	2008-04-08
 228	val_228	2008-04-08
-33	val_33	2008-04-08
-103	val_103	2008-04-08
-367	val_367	2008-04-08
-239	val_239	2008-04-08
-480	val_480	2008-04-08
-202	val_202	2008-04-08
-316	val_316	2008-04-08
 235	val_235	2008-04-08
-80	val_80	2008-04-08
-44	val_44	2008-04-08
-466	val_466	2008-04-08
+239	val_239	2008-04-08
+239	val_239	2008-04-08
+242	val_242	2008-04-08
+242	val_242	2008-04-08
 257	val_257	2008-04-08
-190	val_190	2008-04-08
-114	val_114	2008-04-08
-396	val_396	2008-04-08
-217	val_217	2008-04-08
-125	val_125	2008-04-08
-187	val_187	2008-04-08
-480	val_480	2008-04-08
-491	val_491	2008-04-08
+26	val_26	2008-04-08
+26	val_26	2008-04-08
+260	val_260	2008-04-08
+275	val_275	2008-04-08
+282	val_282	2008-04-08
+282	val_282	2008-04-08
+286	val_286	2008-04-08
 305	val_305	2008-04-08
-444	val_444	2008-04-08
-169	val_169	2008-04-08
+309	val_309	2008-04-08
+309	val_309	2008-04-08
+316	val_316	2008-04-08
+316	val_316	2008-04-08
+316	val_316	2008-04-08
 323	val_323	2008-04-08
-480	val_480	2008-04-08
-136	val_136	2008-04-08
-172	val_172	2008-04-08
-462	val_462	2008-04-08
-26	val_26	2008-04-08
-462	val_462	2008-04-08
+327	val_327	2008-04-08
+327	val_327	2008-04-08
+327	val_327	2008-04-08
+33	val_33	2008-04-08
+338	val_338	2008-04-08
 341	val_341	2008-04-08
-183	val_183	2008-04-08
-84	val_84	2008-04-08
+345	val_345	2008-04-08
+356	val_356	2008-04-08
+367	val_367	2008-04-08
+367	val_367	2008-04-08
 37	val_37	2008-04-08
+37	val_37	2008-04-08
+374	val_374	2008-04-08
+378	val_378	2008-04-08
+389	val_389	2008-04-08
+392	val_392	2008-04-08
+396	val_396	2008-04-08
+396	val_396	2008-04-08
+396	val_396	2008-04-08
+4	val_4	2008-04-08
+400	val_400	2008-04-08
+404	val_404	2008-04-08
+404	val_404	2008-04-08
+411	val_411	2008-04-08
+419	val_419	2008-04-08
+437	val_437	2008-04-08
+44	val_44	2008-04-08
+444	val_444	2008-04-08
 448	val_448	2008-04-08
-194	val_194	2008-04-08
+455	val_455	2008-04-08
+459	val_459	2008-04-08
+459	val_459	2008-04-08
+462	val_462	2008-04-08
+462	val_462	2008-04-08
+466	val_466	2008-04-08
+466	val_466	2008-04-08
+466	val_466	2008-04-08
 477	val_477	2008-04-08
-169	val_169	2008-04-08
-400	val_400	2008-04-08
+480	val_480	2008-04-08
+480	val_480	2008-04-08
+480	val_480	2008-04-08
+484	val_484	2008-04-08
+491	val_491	2008-04-08
+495	val_495	2008-04-08
+51	val_51	2008-04-08
+51	val_51	2008-04-08
+66	val_66	2008-04-08
+77	val_77	2008-04-08
+8	val_8	2008-04-08
+80	val_80	2008-04-08
+84	val_84	2008-04-08
+84	val_84	2008-04-08
+95	val_95	2008-04-08
+95	val_95	2008-04-08
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -416,6 +431,9 @@ STAGE PLANS:
   Stage: Stage-1
     Tez
 #### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
       Vertices:
         Map 1 
             Map Operator Tree:
@@ -426,16 +444,28 @@ STAGE PLANS:
                     expressions: key (type: int), value (type: string), ds (type: string), hr (type: int)
                     outputColumnNames: _col0, _col1, _col2, _col3
                     Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          name: default.srcbucket_mapjoin_n8
+                    Reduce Output Operator
+                      key expressions: _col2 (type: string), _col3 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col2 (type: string), _col3 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: vectorized, llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.srcbucket_mapjoin_n8
 
   Stage: Stage-2
     Dependency Collection
@@ -482,124 +512,124 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
 #### A masked pattern was here ####
-484	val_484	2008-04-08	0
-224	val_224	2008-04-08	0
-213	val_213	2008-04-08	0
-495	val_495	2008-04-08	0
-327	val_327	2008-04-08	0
-338	val_338	2008-04-08	0
-466	val_466	2008-04-08	0
-309	val_309	2008-04-08	0
 0	val_0	2008-04-08	0
-316	val_316	2008-04-08	0
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+103	val_103	2008-04-08	0
+103	val_103	2008-04-08	1
+11	val_11	2008-04-08	1
+114	val_114	2008-04-08	1
+118	val_118	2008-04-08	0
+118	val_118	2008-04-08	0
+125	val_125	2008-04-08	0
+125	val_125	2008-04-08	1
 129	val_129	2008-04-08	0
-4	val_4	2008-04-08	0
+129	val_129	2008-04-08	1
+136	val_136	2008-04-08	1
+143	val_143	2008-04-08	0
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+150	val_150	2008-04-08	1
+158	val_158	2008-04-08	0
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
 169	val_169	2008-04-08	0
-437	val_437	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
+176	val_176	2008-04-08	1
+176	val_176	2008-04-08	1
+183	val_183	2008-04-08	1
 187	val_187	2008-04-08	0
-459	val_459	2008-04-08	0
-103	val_103	2008-04-08	0
+187	val_187	2008-04-08	1
+187	val_187	2008-04-08	1
+19	val_19	2008-04-08	1
+190	val_190	2008-04-08	0
+194	val_194	2008-04-08	1
+202	val_202	2008-04-08	1
 213	val_213	2008-04-08	0
-275	val_275	2008-04-08	0
-404	val_404	2008-04-08	0
-84	val_84	2008-04-08	0
-8	val_8	2008-04-08	0
-172	val_172	2008-04-08	0
-158	val_158	2008-04-08	0
-26	val_26	2008-04-08	0
-327	val_327	2008-04-08	0
-404	val_404	2008-04-08	0
-282	val_282	2008-04-08	0
-316	val_316	2008-04-08	0
-77	val_77	2008-04-08	0
-118	val_118	2008-04-08	0
-419	val_419	2008-04-08	0
-118	val_118	2008-04-08	0
+213	val_213	2008-04-08	0
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
 224	val_224	2008-04-08	0
-389	val_389	2008-04-08	0
+224	val_224	2008-04-08	0
+228	val_228	2008-04-08	1
+235	val_235	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
 242	val_242	2008-04-08	0
 242	val_242	2008-04-08	0
-95	val_95	2008-04-08	0
-143	val_143	2008-04-08	0
-33	val_33	2008-04-08	0
-367	val_367	2008-04-08	0
-480	val_480	2008-04-08	0
-316	val_316	2008-04-08	0
-80	val_80	2008-04-08	0
-466	val_466	2008-04-08	0
-190	val_190	2008-04-08	0
-396	val_396	2008-04-08	0
-125	val_125	2008-04-08	0
-480	val_480	2008-04-08	0
-305	val_305	2008-04-08	0
-169	val_169	2008-04-08	0
-480	val_480	2008-04-08	0
-172	val_172	2008-04-08	0
+257	val_257	2008-04-08	1
+26	val_26	2008-04-08	0
 26	val_26	2008-04-08	0
-341	val_341	2008-04-08	0
-84	val_84	2008-04-08	0
-448	val_448	2008-04-08	0
-477	val_477	2008-04-08	0
-400	val_400	2008-04-08	0
-165	val_165	2008-04-08	1
-150	val_150	2008-04-08	1
-66	val_66	2008-04-08	1
-374	val_374	2008-04-08	1
-37	val_37	2008-04-08	1
-15	val_15	2008-04-08	1
-459	val_459	2008-04-08	1
-396	val_396	2008-04-08	1
-367	val_367	2008-04-08	1
-455	val_455	2008-04-08	1
-345	val_345	2008-04-08	1
-378	val_378	2008-04-08	1
-356	val_356	2008-04-08	1
-125	val_125	2008-04-08	1
-286	val_286	2008-04-08	1
-176	val_176	2008-04-08	1
-51	val_51	2008-04-08	1
-239	val_239	2008-04-08	1
-176	val_176	2008-04-08	1
 260	val_260	2008-04-08	1
-217	val_217	2008-04-08	1
-466	val_466	2008-04-08	1
-411	val_411	2008-04-08	1
-129	val_129	2008-04-08	1
-0	val_0	2008-04-08	1
-165	val_165	2008-04-08	1
-51	val_51	2008-04-08	1
-95	val_95	2008-04-08	1
-187	val_187	2008-04-08	1
-169	val_169	2008-04-08	1
-0	val_0	2008-04-08	1
+275	val_275	2008-04-08	0
+282	val_282	2008-04-08	0
 282	val_282	2008-04-08	1
-15	val_15	2008-04-08	1
-19	val_19	2008-04-08	1
+286	val_286	2008-04-08	1
+305	val_305	2008-04-08	0
+309	val_309	2008-04-08	0
 309	val_309	2008-04-08	1
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+323	val_323	2008-04-08	1
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	0
 327	val_327	2008-04-08	1
+33	val_33	2008-04-08	0
+338	val_338	2008-04-08	0
+341	val_341	2008-04-08	0
+345	val_345	2008-04-08	1
+356	val_356	2008-04-08	1
+367	val_367	2008-04-08	0
+367	val_367	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+374	val_374	2008-04-08	1
+378	val_378	2008-04-08	1
+389	val_389	2008-04-08	0
 392	val_392	2008-04-08	1
+396	val_396	2008-04-08	0
 396	val_396	2008-04-08	1
-11	val_11	2008-04-08	1
-228	val_228	2008-04-08	1
-103	val_103	2008-04-08	1
-239	val_239	2008-04-08	1
-202	val_202	2008-04-08	1
-235	val_235	2008-04-08	1
+396	val_396	2008-04-08	1
+4	val_4	2008-04-08	0
+400	val_400	2008-04-08	0
+404	val_404	2008-04-08	0
+404	val_404	2008-04-08	0
+411	val_411	2008-04-08	1
+419	val_419	2008-04-08	0
+437	val_437	2008-04-08	0
 44	val_44	2008-04-08	1
-257	val_257	2008-04-08	1
-114	val_114	2008-04-08	1
-217	val_217	2008-04-08	1
-187	val_187	2008-04-08	1
-491	val_491	2008-04-08	1
 444	val_444	2008-04-08	1
-323	val_323	2008-04-08	1
-136	val_136	2008-04-08	1
+448	val_448	2008-04-08	0
+455	val_455	2008-04-08	1
+459	val_459	2008-04-08	0
+459	val_459	2008-04-08	1
 462	val_462	2008-04-08	1
 462	val_462	2008-04-08	1
-183	val_183	2008-04-08	1
-37	val_37	2008-04-08	1
-194	val_194	2008-04-08	1
-169	val_169	2008-04-08	1
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	1
+477	val_477	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+484	val_484	2008-04-08	0
+491	val_491	2008-04-08	1
+495	val_495	2008-04-08	0
+51	val_51	2008-04-08	1
+51	val_51	2008-04-08	1
+66	val_66	2008-04-08	1
+77	val_77	2008-04-08	0
+8	val_8	2008-04-08	0
+80	val_80	2008-04-08	0
+84	val_84	2008-04-08	0
+84	val_84	2008-04-08	0
+95	val_95	2008-04-08	0
+95	val_95	2008-04-08	1
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -633,6 +663,9 @@ STAGE PLANS:
   Stage: Stage-1
     Tez
 #### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
       Vertices:
         Map 1 
             Map Operator Tree:
@@ -643,16 +676,28 @@ STAGE PLANS:
                     expressions: key (type: int), value (type: string), ds (type: string), hr (type: int)
                     outputColumnNames: _col0, _col1, _col2, _col3
                     Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          name: default.srcbucket_mapjoin_n8
+                    Reduce Output Operator
+                      key expressions: _col2 (type: string), _col3 (type: int)
+                      sort order: ++
+                      Map-reduce partition columns: _col2 (type: string), _col3 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: vectorized, llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.srcbucket_mapjoin_n8
 
   Stage: Stage-2
     Dependency Collection
@@ -699,242 +744,242 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
 #### A masked pattern was here ####
-484	val_484	2008-04-08	0
-224	val_224	2008-04-08	0
-213	val_213	2008-04-08	0
-495	val_495	2008-04-08	0
-327	val_327	2008-04-08	0
-338	val_338	2008-04-08	0
-466	val_466	2008-04-08	0
-309	val_309	2008-04-08	0
 0	val_0	2008-04-08	0
-316	val_316	2008-04-08	0
-129	val_129	2008-04-08	0
-4	val_4	2008-04-08	0
-169	val_169	2008-04-08	0
-437	val_437	2008-04-08	0
-187	val_187	2008-04-08	0
-459	val_459	2008-04-08	0
+0	val_0	2008-04-08	0
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
 103	val_103	2008-04-08	0
-213	val_213	2008-04-08	0
-275	val_275	2008-04-08	0
-404	val_404	2008-04-08	0
-84	val_84	2008-04-08	0
-8	val_8	2008-04-08	0
-172	val_172	2008-04-08	0
-158	val_158	2008-04-08	0
-26	val_26	2008-04-08	0
-327	val_327	2008-04-08	0
-404	val_404	2008-04-08	0
-282	val_282	2008-04-08	0
-316	val_316	2008-04-08	0
-77	val_77	2008-04-08	0
+103	val_103	2008-04-08	0
+103	val_103	2008-04-08	1
+103	val_103	2008-04-08	1
+11	val_11	2008-04-08	1
+11	val_11	2008-04-08	1
+114	val_114	2008-04-08	1
+114	val_114	2008-04-08	1
 118	val_118	2008-04-08	0
-419	val_419	2008-04-08	0
 118	val_118	2008-04-08	0
-224	val_224	2008-04-08	0
-389	val_389	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-95	val_95	2008-04-08	0
-143	val_143	2008-04-08	0
-33	val_33	2008-04-08	0
-367	val_367	2008-04-08	0
-480	val_480	2008-04-08	0
-316	val_316	2008-04-08	0
-80	val_80	2008-04-08	0
-466	val_466	2008-04-08	0
-190	val_190	2008-04-08	0
-396	val_396	2008-04-08	0
-125	val_125	2008-04-08	0
-480	val_480	2008-04-08	0
-305	val_305	2008-04-08	0
-169	val_169	2008-04-08	0
-480	val_480	2008-04-08	0
-172	val_172	2008-04-08	0
-26	val_26	2008-04-08	0
-341	val_341	2008-04-08	0
-84	val_84	2008-04-08	0
-448	val_448	2008-04-08	0
-477	val_477	2008-04-08	0
-400	val_400	2008-04-08	0
-484	val_484	2008-04-08	0
-224	val_224	2008-04-08	0
-213	val_213	2008-04-08	0
-495	val_495	2008-04-08	0
-327	val_327	2008-04-08	0
-338	val_338	2008-04-08	0
-466	val_466	2008-04-08	0
-309	val_309	2008-04-08	0
-0	val_0	2008-04-08	0
-316	val_316	2008-04-08	0
-129	val_129	2008-04-08	0
-4	val_4	2008-04-08	0
-169	val_169	2008-04-08	0
-437	val_437	2008-04-08	0
-187	val_187	2008-04-08	0
-459	val_459	2008-04-08	0
-103	val_103	2008-04-08	0
-213	val_213	2008-04-08	0
-275	val_275	2008-04-08	0
-404	val_404	2008-04-08	0
-84	val_84	2008-04-08	0
-8	val_8	2008-04-08	0
-172	val_172	2008-04-08	0
-158	val_158	2008-04-08	0
-26	val_26	2008-04-08	0
-327	val_327	2008-04-08	0
-404	val_404	2008-04-08	0
-282	val_282	2008-04-08	0
-316	val_316	2008-04-08	0
-77	val_77	2008-04-08	0
 118	val_118	2008-04-08	0
-419	val_419	2008-04-08	0
 118	val_118	2008-04-08	0
-224	val_224	2008-04-08	0
-389	val_389	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-95	val_95	2008-04-08	0
-143	val_143	2008-04-08	0
-33	val_33	2008-04-08	0
-367	val_367	2008-04-08	0
-480	val_480	2008-04-08	0
-316	val_316	2008-04-08	0
-80	val_80	2008-04-08	0
-466	val_466	2008-04-08	0
-190	val_190	2008-04-08	0
-396	val_396	2008-04-08	0
 125	val_125	2008-04-08	0
-480	val_480	2008-04-08	0
-305	val_305	2008-04-08	0
+125	val_125	2008-04-08	0
+125	val_125	2008-04-08	1
+125	val_125	2008-04-08	1
+129	val_129	2008-04-08	0
+129	val_129	2008-04-08	0
+129	val_129	2008-04-08	1
+129	val_129	2008-04-08	1
+136	val_136	2008-04-08	1
+136	val_136	2008-04-08	1
+143	val_143	2008-04-08	0
+143	val_143	2008-04-08	0
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+150	val_150	2008-04-08	1
+150	val_150	2008-04-08	1
+158	val_158	2008-04-08	0
+158	val_158	2008-04-08	0
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
 169	val_169	2008-04-08	0
-480	val_480	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
 172	val_172	2008-04-08	0
-26	val_26	2008-04-08	0
-341	val_341	2008-04-08	0
-84	val_84	2008-04-08	0
-448	val_448	2008-04-08	0
-477	val_477	2008-04-08	0
-400	val_400	2008-04-08	0
-165	val_165	2008-04-08	1
-150	val_150	2008-04-08	1
-66	val_66	2008-04-08	1
-374	val_374	2008-04-08	1
-37	val_37	2008-04-08	1
-15	val_15	2008-04-08	1
-459	val_459	2008-04-08	1
-396	val_396	2008-04-08	1
-367	val_367	2008-04-08	1
-455	val_455	2008-04-08	1
-345	val_345	2008-04-08	1
-378	val_378	2008-04-08	1
-356	val_356	2008-04-08	1
-125	val_125	2008-04-08	1
-286	val_286	2008-04-08	1
 176	val_176	2008-04-08	1
-51	val_51	2008-04-08	1
-239	val_239	2008-04-08	1
 176	val_176	2008-04-08	1
-260	val_260	2008-04-08	1
-217	val_217	2008-04-08	1
-466	val_466	2008-04-08	1
-411	val_411	2008-04-08	1
-129	val_129	2008-04-08	1
-0	val_0	2008-04-08	1
-165	val_165	2008-04-08	1
-51	val_51	2008-04-08	1
-95	val_95	2008-04-08	1
+176	val_176	2008-04-08	1
+176	val_176	2008-04-08	1
+183	val_183	2008-04-08	1
+183	val_183	2008-04-08	1
+187	val_187	2008-04-08	0
+187	val_187	2008-04-08	0
+187	val_187	2008-04-08	1
+187	val_187	2008-04-08	1
+187	val_187	2008-04-08	1
 187	val_187	2008-04-08	1
-169	val_169	2008-04-08	1
-0	val_0	2008-04-08	1
-282	val_282	2008-04-08	1
-15	val_15	2008-04-08	1
 19	val_19	2008-04-08	1
-309	val_309	2008-04-08	1
-327	val_327	2008-04-08	1
-392	val_392	2008-04-08	1
-396	val_396	2008-04-08	1
-11	val_11	2008-04-08	1
-228	val_228	2008-04-08	1
-103	val_103	2008-04-08	1
-239	val_239	2008-04-08	1
+19	val_19	2008-04-08	1
+190	val_190	2008-04-08	0
+190	val_190	2008-04-08	0
+194	val_194	2008-04-08	1
+194	val_194	2008-04-08	1
 202	val_202	2008-04-08	1
-235	val_235	2008-04-08	1
-44	val_44	2008-04-08	1
-257	val_257	2008-04-08	1
-114	val_114	2008-04-08	1
+202	val_202	2008-04-08	1
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
 217	val_217	2008-04-08	1
-187	val_187	2008-04-08	1
-491	val_491	2008-04-08	1
-444	val_444	2008-04-08	1
-323	val_323	2008-04-08	1
-136	val_136	2008-04-08	1
-462	val_462	2008-04-08	1
-462	val_462	2008-04-08	1
-183	val_183	2008-04-08	1
-37	val_37	2008-04-08	1
-194	val_194	2008-04-08	1
-169	val_169	2008-04-08	1
-165	val_165	2008-04-08	1
-150	val_150	2008-04-08	1
-66	val_66	2008-04-08	1
-374	val_374	2008-04-08	1
-37	val_37	2008-04-08	1
-15	val_15	2008-04-08	1
-459	val_459	2008-04-08	1
-396	val_396	2008-04-08	1
-367	val_367	2008-04-08	1
-455	val_455	2008-04-08	1
-345	val_345	2008-04-08	1
-378	val_378	2008-04-08	1
-356	val_356	2008-04-08	1
-125	val_125	2008-04-08	1
-286	val_286	2008-04-08	1
-176	val_176	2008-04-08	1
-51	val_51	2008-04-08	1
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
+228	val_228	2008-04-08	1
+228	val_228	2008-04-08	1
+235	val_235	2008-04-08	1
+235	val_235	2008-04-08	1
 239	val_239	2008-04-08	1
-176	val_176	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
+257	val_257	2008-04-08	1
+257	val_257	2008-04-08	1
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
 260	val_260	2008-04-08	1
-217	val_217	2008-04-08	1
-466	val_466	2008-04-08	1
-411	val_411	2008-04-08	1
-129	val_129	2008-04-08	1
-0	val_0	2008-04-08	1
-165	val_165	2008-04-08	1
-51	val_51	2008-04-08	1
-95	val_95	2008-04-08	1
-187	val_187	2008-04-08	1
-169	val_169	2008-04-08	1
-0	val_0	2008-04-08	1
+260	val_260	2008-04-08	1
+275	val_275	2008-04-08	0
+275	val_275	2008-04-08	0
+282	val_282	2008-04-08	0
+282	val_282	2008-04-08	0
 282	val_282	2008-04-08	1
-15	val_15	2008-04-08	1
-19	val_19	2008-04-08	1
+282	val_282	2008-04-08	1
+286	val_286	2008-04-08	1
+286	val_286	2008-04-08	1
+305	val_305	2008-04-08	0
+305	val_305	2008-04-08	0
+309	val_309	2008-04-08	0
+309	val_309	2008-04-08	0
+309	val_309	2008-04-08	1
 309	val_309	2008-04-08	1
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+323	val_323	2008-04-08	1
+323	val_323	2008-04-08	1
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	1
 327	val_327	2008-04-08	1
+33	val_33	2008-04-08	0
+33	val_33	2008-04-08	0
+338	val_338	2008-04-08	0
+338	val_338	2008-04-08	0
+341	val_341	2008-04-08	0
+341	val_341	2008-04-08	0
+345	val_345	2008-04-08	1
+345	val_345	2008-04-08	1
+356	val_356	2008-04-08	1
+356	val_356	2008-04-08	1
+367	val_367	2008-04-08	0
+367	val_367	2008-04-08	0
+367	val_367	2008-04-08	1
+367	val_367	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+374	val_374	2008-04-08	1
+374	val_374	2008-04-08	1
+378	val_378	2008-04-08	1
+378	val_378	2008-04-08	1
+389	val_389	2008-04-08	0
+389	val_389	2008-04-08	0
 392	val_392	2008-04-08	1
+392	val_392	2008-04-08	1
+396	val_396	2008-04-08	0
+396	val_396	2008-04-08	0
 396	val_396	2008-04-08	1
-11	val_11	2008-04-08	1
-228	val_228	2008-04-08	1
-103	val_103	2008-04-08	1
-239	val_239	2008-04-08	1
-202	val_202	2008-04-08	1
-235	val_235	2008-04-08	1
+396	val_396	2008-04-08	1
+396	val_396	2008-04-08	1
+396	val_396	2008-04-08	1
+4	val_4	2008-04-08	0
+4	val_4	2008-04-08	0
+400	val_400	2008-04-08	0
+400	val_400	2008-04-08	0
+404	val_404	2008-04-08	0
+404	val_404	2008-04-08	0
+404	val_404	2008-04-08	0
+404	val_404	2008-04-08	0
+411	val_411	2008-04-08	1
+411	val_411	2008-04-08	1
+419	val_419	2008-04-08	0
+419	val_419	2008-04-08	0
+437	val_437	2008-04-08	0
+437	val_437	2008-04-08	0
+44	val_44	2008-04-08	1
 44	val_44	2008-04-08	1
-257	val_257	2008-04-08	1
-114	val_114	2008-04-08	1
-217	val_217	2008-04-08	1
-187	val_187	2008-04-08	1
-491	val_491	2008-04-08	1
 444	val_444	2008-04-08	1
-323	val_323	2008-04-08	1
-136	val_136	2008-04-08	1
+444	val_444	2008-04-08	1
+448	val_448	2008-04-08	0
+448	val_448	2008-04-08	0
+455	val_455	2008-04-08	1
+455	val_455	2008-04-08	1
+459	val_459	2008-04-08	0
+459	val_459	2008-04-08	0
+459	val_459	2008-04-08	1
+459	val_459	2008-04-08	1
 462	val_462	2008-04-08	1
 462	val_462	2008-04-08	1
-183	val_183	2008-04-08	1
-37	val_37	2008-04-08	1
-194	val_194	2008-04-08	1
-169	val_169	2008-04-08	1
+462	val_462	2008-04-08	1
+462	val_462	2008-04-08	1
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	1
+466	val_466	2008-04-08	1
+477	val_477	2008-04-08	0
+477	val_477	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+484	val_484	2008-04-08	0
+484	val_484	2008-04-08	0
+491	val_491	2008-04-08	1
+491	val_491	2008-04-08	1
+495	val_495	2008-04-08	0
+495	val_495	2008-04-08	0
+51	val_51	2008-04-08	1
+51	val_51	2008-04-08	1
+51	val_51	2008-04-08	1
+51	val_51	2008-04-08	1
+66	val_66	2008-04-08	1
+66	val_66	2008-04-08	1
+77	val_77	2008-04-08	0
+77	val_77	2008-04-08	0
+8	val_8	2008-04-08	0
+8	val_8	2008-04-08	0
+80	val_80	2008-04-08	0
+80	val_80	2008-04-08	0
+84	val_84	2008-04-08	0
+84	val_84	2008-04-08	0
+84	val_84	2008-04-08	0
+84	val_84	2008-04-08	0
+95	val_95	2008-04-08	0
+95	val_95	2008-04-08	0
+95	val_95	2008-04-08	1
+95	val_95	2008-04-08	1
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -1041,124 +1086,124 @@ POSTHOOK: query: select * from srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8
 #### A masked pattern was here ####
-8	val_8
-33	val_33
-37	val_37
-37	val_37
-51	val_51
-51	val_51
-103	val_103
-103	val_103
-125	val_125
-125	val_125
-165	val_165
-165	val_165
-194	val_194
-260	val_260
-275	val_275
-305	val_305
-323	val_323
-411	val_411
-455	val_455
-466	val_466
-466	val_466
-466	val_466
-484	val_484
 0	val_0
 0	val_0
 0	val_0
-4	val_4
+103	val_103
+103	val_103
 11	val_11
-19	val_19
-26	val_26
-26	val_26
-44	val_44
-77	val_77
+114	val_114
 118	val_118
 118	val_118
-158	val_158
-217	val_217
-217	val_217
-316	val_316
-316	val_316
-316	val_316
-327	val_327
-327	val_327
-327	val_327
-338	val_338
-356	val_356
-378	val_378
-389	val_389
-404	val_404
-404	val_404
-419	val_419
-444	val_444
-462	val_462
-462	val_462
-15	val_15
-15	val_15
-80	val_80
-84	val_84
-84	val_84
-95	val_95
-95	val_95
+125	val_125
+125	val_125
 129	val_129
 129	val_129
+136	val_136
+143	val_143
+15	val_15
+15	val_15
+150	val_150
+158	val_158
+165	val_165
+165	val_165
 169	val_169
 169	val_169
 169	val_169
 169	val_169
-239	val_239
-239	val_239
-282	val_282
-282	val_282
-367	val_367
-367	val_367
-66	val_66
-114	val_114
-136	val_136
-143	val_143
 172	val_172
 172	val_172
-190	val_190
-242	val_242
-242	val_242
-286	val_286
-345	val_345
-400	val_400
-437	val_437
-448	val_448
-459	val_459
-459	val_459
-150	val_150
 176	val_176
 176	val_176
 183	val_183
 187	val_187
 187	val_187
 187	val_187
+19	val_19
+190	val_190
+194	val_194
 202	val_202
 213	val_213
 213	val_213
+217	val_217
+217	val_217
 224	val_224
 224	val_224
 228	val_228
 235	val_235
+239	val_239
+239	val_239
+242	val_242
+242	val_242
 257	val_257
+26	val_26
+26	val_26
+260	val_260
+275	val_275
+282	val_282
+282	val_282
+286	val_286
+305	val_305
 309	val_309
 309	val_309
+316	val_316
+316	val_316
+316	val_316
+323	val_323
+327	val_327
+327	val_327
+327	val_327
+33	val_33
+338	val_338
 341	val_341
+345	val_345
+356	val_356
+367	val_367
+367	val_367
+37	val_37
+37	val_37
 374	val_374
+378	val_378
+389	val_389
 392	val_392
 396	val_396
 396	val_396
 396	val_396
+4	val_4
+400	val_400
+404	val_404
+404	val_404
+411	val_411
+419	val_419
+437	val_437
+44	val_44
+444	val_444
+448	val_448
+455	val_455
+459	val_459
+459	val_459
+462	val_462
+462	val_462
+466	val_466
+466	val_466
+466	val_466
 477	val_477
 480	val_480
 480	val_480
 480	val_480
+484	val_484
 491	val_491
 495	val_495
+51	val_51
+51	val_51
+66	val_66
+77	val_77
+8	val_8
+80	val_80
+84	val_84
+84	val_84
+95	val_95
+95	val_95
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -1206,250 +1251,22 @@ STAGE PLANS:
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
-                      key expressions: _col0 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: int)
-                      Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string), _col2 (type: string)
-            Execution mode: vectorized, llap
-            LLAP IO: no inputs
-        Reducer 2 
-            Execution mode: vectorized, llap
-            Reduce Operator Tree:
-              Select Operator
-                expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string)
-                outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                      name: default.srcbucket_mapjoin_n8
-
-  Stage: Stage-2
-    Dependency Collection
-
-  Stage: Stage-0
-    Move Operator
-      tables:
-          partition:
-            ds 
-          replace: false
-          table:
-              input format: org.apache.hadoop.mapred.TextInputFormat
-              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-              name: default.srcbucket_mapjoin_n8
-
-  Stage: Stage-3
-    Stats Work
-      Basic Stats Work:
-
-PREHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8
-PREHOOK: type: QUERY
-PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
-PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
-POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08
-POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ]
-PREHOOK: query: select * from srcbucket_mapjoin_n8
-PREHOOK: type: QUERY
-PREHOOK: Input: default@srcbucket_mapjoin_n8
-PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08
-#### A masked pattern was here ####
-POSTHOOK: query: select * from srcbucket_mapjoin_n8
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@srcbucket_mapjoin_n8
-POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08
-#### A masked pattern was here ####
-8	val_8	2008-04-08
-33	val_33	2008-04-08
-37	val_37	2008-04-08
-37	val_37	2008-04-08
-51	val_51	2008-04-08
-51	val_51	2008-04-08
-103	val_103	2008-04-08
-103	val_103	2008-04-08
-125	val_125	2008-04-08
-125	val_125	2008-04-08
-165	val_165	2008-04-08
-165	val_165	2008-04-08
-194	val_194	2008-04-08
-260	val_260	2008-04-08
-275	val_275	2008-04-08
-305	val_305	2008-04-08
-323	val_323	2008-04-08
-411	val_411	2008-04-08
-455	val_455	2008-04-08
-466	val_466	2008-04-08
-466	val_466	2008-04-08
-466	val_466	2008-04-08
-484	val_484	2008-04-08
-0	val_0	2008-04-08
-0	val_0	2008-04-08
-0	val_0	2008-04-08
-4	val_4	2008-04-08
-11	val_11	2008-04-08
-19	val_19	2008-04-08
-26	val_26	2008-04-08
-26	val_26	2008-04-08
-44	val_44	2008-04-08
-77	val_77	2008-04-08
-118	val_118	2008-04-08
-118	val_118	2008-04-08
-158	val_158	2008-04-08
-217	val_217	2008-04-08
-217	val_217	2008-04-08
-316	val_316	2008-04-08
-316	val_316	2008-04-08
-316	val_316	2008-04-08
-327	val_327	2008-04-08
-327	val_327	2008-04-08
-327	val_327	2008-04-08
-338	val_338	2008-04-08
-356	val_356	2008-04-08
-378	val_378	2008-04-08
-389	val_389	2008-04-08
-404	val_404	2008-04-08
-404	val_404	2008-04-08
-419	val_419	2008-04-08
-444	val_444	2008-04-08
-462	val_462	2008-04-08
-462	val_462	2008-04-08
-15	val_15	2008-04-08
-15	val_15	2008-04-08
-80	val_80	2008-04-08
-84	val_84	2008-04-08
-84	val_84	2008-04-08
-95	val_95	2008-04-08
-95	val_95	2008-04-08
-129	val_129	2008-04-08
-129	val_129	2008-04-08
-169	val_169	2008-04-08
-169	val_169	2008-04-08
-169	val_169	2008-04-08
-169	val_169	2008-04-08
-239	val_239	2008-04-08
-239	val_239	2008-04-08
-282	val_282	2008-04-08
-282	val_282	2008-04-08
-367	val_367	2008-04-08
-367	val_367	2008-04-08
-66	val_66	2008-04-08
-114	val_114	2008-04-08
-136	val_136	2008-04-08
-143	val_143	2008-04-08
-172	val_172	2008-04-08
-172	val_172	2008-04-08
-190	val_190	2008-04-08
-242	val_242	2008-04-08
-242	val_242	2008-04-08
-286	val_286	2008-04-08
-345	val_345	2008-04-08
-400	val_400	2008-04-08
-437	val_437	2008-04-08
-448	val_448	2008-04-08
-459	val_459	2008-04-08
-459	val_459	2008-04-08
-150	val_150	2008-04-08
-176	val_176	2008-04-08
-176	val_176	2008-04-08
-183	val_183	2008-04-08
-187	val_187	2008-04-08
-187	val_187	2008-04-08
-187	val_187	2008-04-08
-202	val_202	2008-04-08
-213	val_213	2008-04-08
-213	val_213	2008-04-08
-224	val_224	2008-04-08
-224	val_224	2008-04-08
-228	val_228	2008-04-08
-235	val_235	2008-04-08
-257	val_257	2008-04-08
-309	val_309	2008-04-08
-309	val_309	2008-04-08
-341	val_341	2008-04-08
-374	val_374	2008-04-08
-392	val_392	2008-04-08
-396	val_396	2008-04-08
-396	val_396	2008-04-08
-396	val_396	2008-04-08
-477	val_477	2008-04-08
-480	val_480	2008-04-08
-480	val_480	2008-04-08
-480	val_480	2008-04-08
-491	val_491	2008-04-08
-495	val_495	2008-04-08
-PREHOOK: query: drop table srcbucket_mapjoin_n8
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@srcbucket_mapjoin_n8
-PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: drop table srcbucket_mapjoin_n8
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@srcbucket_mapjoin_n8
-POSTHOOK: Output: default@srcbucket_mapjoin_n8
-PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@srcbucket_mapjoin_n8
-PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
-PREHOOK: type: QUERY
-PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
-PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
-STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
-  Stage-0 depends on stages: Stage-2
-  Stage-3 depends on stages: Stage-0
-
-STAGE PLANS:
-  Stage: Stage-1
-    Tez
-#### A masked pattern was here ####
-      Edges:
-        Reducer 2 <- Map 1 (SIMPLE_EDGE)
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: srcbucket_mapjoin_n8__temp_table_for_load_data__
-                  Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
-                  Select Operator
-                    expressions: key (type: int), value (type: string), ds (type: string), hr (type: int)
-                    outputColumnNames: _col0, _col1, _col2, _col3
-                    Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: int)
-                      Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int)
+                      key expressions: _col2 (type: string), _bucket_number (type: string), _col0 (type: int)
+                      sort order: +++
+                      Map-reduce partition columns: _col2 (type: string)
+                      value expressions: _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: vectorized, llap
             Reduce Operator Tree:
               Select Operator
-                expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int)
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
+                expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._bucket_number (type: string)
+                outputColumnNames: _col0, _col1, _col2, _bucket_number
                 File Output Operator
                   compressed: false
-                  Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
+                  Dp Sort State: PARTITION_BUCKET_SORTED
+                  Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1464,7 +1281,6 @@ STAGE PLANS:
       tables:
           partition:
             ds 
-            hr 
           replace: false
           table:
               input format: org.apache.hadoop.mapred.TextInputFormat
@@ -1476,149 +1292,144 @@ STAGE PLANS:
     Stats Work
       Basic Stats Work:
 
-PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
+PREHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8
 PREHOOK: type: QUERY
 PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
+POSTHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
-POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
-POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
-POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ]
-POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08
+POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ]
 PREHOOK: query: select * from srcbucket_mapjoin_n8
 PREHOOK: type: QUERY
 PREHOOK: Input: default@srcbucket_mapjoin_n8
-PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
-PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
+PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08
 #### A masked pattern was here ####
 POSTHOOK: query: select * from srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8
-POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
-POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
+POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08
 #### A masked pattern was here ####
-8	val_8	2008-04-08	0
-33	val_33	2008-04-08	0
-103	val_103	2008-04-08	0
-125	val_125	2008-04-08	0
-275	val_275	2008-04-08	0
-305	val_305	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-484	val_484	2008-04-08	0
-0	val_0	2008-04-08	0
-4	val_4	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-77	val_77	2008-04-08	0
-118	val_118	2008-04-08	0
-118	val_118	2008-04-08	0
-158	val_158	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-338	val_338	2008-04-08	0
-389	val_389	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
-419	val_419	2008-04-08	0
-80	val_80	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-95	val_95	2008-04-08	0
-129	val_129	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-282	val_282	2008-04-08	0
-367	val_367	2008-04-08	0
-143	val_143	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-190	val_190	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-400	val_400	2008-04-08	0
-437	val_437	2008-04-08	0
-448	val_448	2008-04-08	0
-459	val_459	2008-04-08	0
-187	val_187	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-309	val_309	2008-04-08	0
-341	val_341	2008-04-08	0
-396	val_396	2008-04-08	0
-477	val_477	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-495	val_495	2008-04-08	0
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-103	val_103	2008-04-08	1
-125	val_125	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-194	val_194	2008-04-08	1
-260	val_260	2008-04-08	1
-323	val_323	2008-04-08	1
-411	val_411	2008-04-08	1
-455	val_455	2008-04-08	1
-466	val_466	2008-04-08	1
-0	val_0	2008-04-08	1
-0	val_0	2008-04-08	1
-11	val_11	2008-04-08	1
-19	val_19	2008-04-08	1
-44	val_44	2008-04-08	1
-217	val_217	2008-04-08	1
-217	val_217	2008-04-08	1
-327	val_327	2008-04-08	1
-356	val_356	2008-04-08	1
-378	val_378	2008-04-08	1
-444	val_444	2008-04-08	1
-462	val_462	2008-04-08	1
-462	val_462	2008-04-08	1
-15	val_15	2008-04-08	1
-15	val_15	2008-04-08	1
-95	val_95	2008-04-08	1
-129	val_129	2008-04-08	1
-169	val_169	2008-04-08	1
-169	val_169	2008-04-08	1
-239	val_239	2008-04-08	1
-239	val_239	2008-04-08	1
-282	val_282	2008-04-08	1
-367	val_367	2008-04-08	1
-66	val_66	2008-04-08	1
-114	val_114	2008-04-08	1
-136	val_136	2008-04-08	1
-286	val_286	2008-04-08	1
-345	val_345	2008-04-08	1
-459	val_459	2008-04-08	1
-150	val_150	2008-04-08	1
-176	val_176	2008-04-08	1
-176	val_176	2008-04-08	1
-183	val_183	2008-04-08	1
-187	val_187	2008-04-08	1
-187	val_187	2008-04-08	1
-202	val_202	2008-04-08	1
-228	val_228	2008-04-08	1
-235	val_235	2008-04-08	1
-257	val_257	2008-04-08	1
-309	val_309	2008-04-08	1
-374	val_374	2008-04-08	1
-392	val_392	2008-04-08	1
-396	val_396	2008-04-08	1
-396	val_396	2008-04-08	1
-491	val_491	2008-04-08	1
+0	val_0	2008-04-08
+0	val_0	2008-04-08
+0	val_0	2008-04-08
+103	val_103	2008-04-08
+103	val_103	2008-04-08
+11	val_11	2008-04-08
+114	val_114	2008-04-08
+118	val_118	2008-04-08
+118	val_118	2008-04-08
+125	val_125	2008-04-08
+125	val_125	2008-04-08
+129	val_129	2008-04-08
+129	val_129	2008-04-08
+136	val_136	2008-04-08
+143	val_143	2008-04-08
+15	val_15	2008-04-08
+15	val_15	2008-04-08
+150	val_150	2008-04-08
+158	val_158	2008-04-08
+165	val_165	2008-04-08
+165	val_165	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+169	val_169	2008-04-08
+172	val_172	2008-04-08
+172	val_172	2008-04-08
+176	val_176	2008-04-08
+176	val_176	2008-04-08
+183	val_183	2008-04-08
+187	val_187	2008-04-08
+187	val_187	2008-04-08
+187	val_187	2008-04-08
+19	val_19	2008-04-08
+190	val_190	2008-04-08
+194	val_194	2008-04-08
+202	val_202	2008-04-08
+213	val_213	2008-04-08
+213	val_213	2008-04-08
+217	val_217	2008-04-08
+217	val_217	2008-04-08
+224	val_224	2008-04-08
+224	val_224	2008-04-08
+228	val_228	2008-04-08
+235	val_235	2008-04-08
+239	val_239	2008-04-08
+239	val_239	2008-04-08
+242	val_242	2008-04-08
+242	val_242	2008-04-08
+257	val_257	2008-04-08
+26	val_26	2008-04-08
+26	val_26	2008-04-08
+260	val_260	2008-04-08
+275	val_275	2008-04-08
+282	val_282	2008-04-08
+282	val_282	2008-04-08
+286	val_286	2008-04-08
+305	val_305	2008-04-08
+309	val_309	2008-04-08
+309	val_309	2008-04-08
+316	val_316	2008-04-08
+316	val_316	2008-04-08
+316	val_316	2008-04-08
+323	val_323	2008-04-08
+327	val_327	2008-04-08
+327	val_327	2008-04-08
+327	val_327	2008-04-08
+33	val_33	2008-04-08
+338	val_338	2008-04-08
+341	val_341	2008-04-08
+345	val_345	2008-04-08
+356	val_356	2008-04-08
+367	val_367	2008-04-08
+367	val_367	2008-04-08
+37	val_37	2008-04-08
+37	val_37	2008-04-08
+374	val_374	2008-04-08
+378	val_378	2008-04-08
+389	val_389	2008-04-08
+392	val_392	2008-04-08
+396	val_396	2008-04-08
+396	val_396	2008-04-08
+396	val_396	2008-04-08
+4	val_4	2008-04-08
+400	val_400	2008-04-08
+404	val_404	2008-04-08
+404	val_404	2008-04-08
+411	val_411	2008-04-08
+419	val_419	2008-04-08
+437	val_437	2008-04-08
+44	val_44	2008-04-08
+444	val_444	2008-04-08
+448	val_448	2008-04-08
+455	val_455	2008-04-08
+459	val_459	2008-04-08
+459	val_459	2008-04-08
+462	val_462	2008-04-08
+462	val_462	2008-04-08
+466	val_466	2008-04-08
+466	val_466	2008-04-08
+466	val_466	2008-04-08
+477	val_477	2008-04-08
+480	val_480	2008-04-08
+480	val_480	2008-04-08
+480	val_480	2008-04-08
+484	val_484	2008-04-08
+491	val_491	2008-04-08
+495	val_495	2008-04-08
+51	val_51	2008-04-08
+51	val_51	2008-04-08
+66	val_66	2008-04-08
+77	val_77	2008-04-08
+8	val_8	2008-04-08
+80	val_80	2008-04-08
+84	val_84	2008-04-08
+84	val_84	2008-04-08
+95	val_95	2008-04-08
+95	val_95	2008-04-08
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -1635,11 +1446,11 @@ POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partit
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@srcbucket_mapjoin_n8
-PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
+PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
 PREHOOK: type: QUERY
 PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
+POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 STAGE DEPENDENCIES:
@@ -1660,29 +1471,28 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: srcbucket_mapjoin_n8__temp_table_for_load_data__
-                  Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
                   Select Operator
                     expressions: key (type: int), value (type: string), ds (type: string), hr (type: int)
                     outputColumnNames: _col0, _col1, _col2, _col3
-                    Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
-                      key expressions: _col0 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: int)
-                      Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int)
+                      key expressions: _col2 (type: string), _col3 (type: int), _bucket_number (type: string), _col0 (type: int)
+                      sort order: ++++
+                      Map-reduce partition columns: _col2 (type: string), _col3 (type: int)
+                      value expressions: _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: vectorized, llap
             Reduce Operator Tree:
               Select Operator
-                expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int)
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
+                expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._bucket_number (type: string)
+                outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number
                 File Output Operator
                   compressed: false
-                  Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
+                  Dp Sort State: PARTITION_BUCKET_SORTED
+                  Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1709,11 +1519,11 @@ STAGE PLANS:
     Stats Work
       Basic Stats Work:
 
-PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
+PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
 PREHOOK: type: QUERY
 PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
+POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
@@ -1734,242 +1544,124 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
 #### A masked pattern was here ####
-8	val_8	2008-04-08	0
-8	val_8	2008-04-08	0
-33	val_33	2008-04-08	0
-33	val_33	2008-04-08	0
-103	val_103	2008-04-08	0
-103	val_103	2008-04-08	0
-125	val_125	2008-04-08	0
-125	val_125	2008-04-08	0
-275	val_275	2008-04-08	0
-275	val_275	2008-04-08	0
-305	val_305	2008-04-08	0
-305	val_305	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-484	val_484	2008-04-08	0
-484	val_484	2008-04-08	0
 0	val_0	2008-04-08	0
-0	val_0	2008-04-08	0
-4	val_4	2008-04-08	0
-4	val_4	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-77	val_77	2008-04-08	0
-77	val_77	2008-04-08	0
-118	val_118	2008-04-08	0
-118	val_118	2008-04-08	0
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+103	val_103	2008-04-08	0
+103	val_103	2008-04-08	1
+11	val_11	2008-04-08	1
+114	val_114	2008-04-08	1
 118	val_118	2008-04-08	0
 118	val_118	2008-04-08	0
-158	val_158	2008-04-08	0
-158	val_158	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-338	val_338	2008-04-08	0
-338	val_338	2008-04-08	0
-389	val_389	2008-04-08	0
-389	val_389	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
-419	val_419	2008-04-08	0
-419	val_419	2008-04-08	0
-80	val_80	2008-04-08	0
-80	val_80	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-95	val_95	2008-04-08	0
-95	val_95	2008-04-08	0
-129	val_129	2008-04-08	0
+125	val_125	2008-04-08	0
+125	val_125	2008-04-08	1
 129	val_129	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-282	val_282	2008-04-08	0
-282	val_282	2008-04-08	0
-367	val_367	2008-04-08	0
-367	val_367	2008-04-08	0
-143	val_143	2008-04-08	0
+129	val_129	2008-04-08	1
+136	val_136	2008-04-08	1
 143	val_143	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-190	val_190	2008-04-08	0
-190	val_190	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-400	val_400	2008-04-08	0
-400	val_400	2008-04-08	0
-437	val_437	2008-04-08	0
-437	val_437	2008-04-08	0
-448	val_448	2008-04-08	0
-448	val_448	2008-04-08	0
-459	val_459	2008-04-08	0
-459	val_459	2008-04-08	0
-187	val_187	2008-04-08	0
-187	val_187	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-309	val_309	2008-04-08	0
-309	val_309	2008-04-08	0
-341	val_341	2008-04-08	0
-341	val_341	2008-04-08	0
-396	val_396	2008-04-08	0
-396	val_396	2008-04-08	0
-477	val_477	2008-04-08	0
-477	val_477	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-495	val_495	2008-04-08	0
-495	val_495	2008-04-08	0
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-103	val_103	2008-04-08	1
-103	val_103	2008-04-08	1
-125	val_125	2008-04-08	1
-125	val_125	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-194	val_194	2008-04-08	1
-194	val_194	2008-04-08	1
-260	val_260	2008-04-08	1
-260	val_260	2008-04-08	1
-323	val_323	2008-04-08	1
-323	val_323	2008-04-08	1
-411	val_411	2008-04-08	1
-411	val_411	2008-04-08	1
-455	val_455	2008-04-08	1
-455	val_455	2008-04-08	1
-466	val_466	2008-04-08	1
-466	val_466	2008-04-08	1
-0	val_0	2008-04-08	1
-0	val_0	2008-04-08	1
-0	val_0	2008-04-08	1
-0	val_0	2008-04-08	1
-11	val_11	2008-04-08	1
-11	val_11	2008-04-08	1
-19	val_19	2008-04-08	1
-19	val_19	2008-04-08	1
-44	val_44	2008-04-08	1
-44	val_44	2008-04-08	1
-217	val_217	2008-04-08	1
-217	val_217	2008-04-08	1
-217	val_217	2008-04-08	1
-217	val_217	2008-04-08	1
-327	val_327	2008-04-08	1
-327	val_327	2008-04-08	1
-356	val_356	2008-04-08	1
-356	val_356	2008-04-08	1
-378	val_378	2008-04-08	1
-378	val_378	2008-04-08	1
-444	val_444	2008-04-08	1
-444	val_444	2008-04-08	1
-462	val_462	2008-04-08	1
-462	val_462	2008-04-08	1
-462	val_462	2008-04-08	1
-462	val_462	2008-04-08	1
-15	val_15	2008-04-08	1
 15	val_15	2008-04-08	1
 15	val_15	2008-04-08	1
-15	val_15	2008-04-08	1
-95	val_95	2008-04-08	1
-95	val_95	2008-04-08	1
-129	val_129	2008-04-08	1
-129	val_129	2008-04-08	1
-169	val_169	2008-04-08	1
-169	val_169	2008-04-08	1
+150	val_150	2008-04-08	1
+158	val_158	2008-04-08	0
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	0
 169	val_169	2008-04-08	1
 169	val_169	2008-04-08	1
-239	val_239	2008-04-08	1
-239	val_239	2008-04-08	1
-239	val_239	2008-04-08	1
-239	val_239	2008-04-08	1
-282	val_282	2008-04-08	1
-282	val_282	2008-04-08	1
-367	val_367	2008-04-08	1
-367	val_367	2008-04-08	1
-66	val_66	2008-04-08	1
-66	val_66	2008-04-08	1
-114	val_114	2008-04-08	1
-114	val_114	2008-04-08	1
-136	val_136	2008-04-08	1
-136	val_136	2008-04-08	1
-286	val_286	2008-04-08	1
-286	val_286	2008-04-08	1
-345	val_345	2008-04-08	1
-345	val_345	2008-04-08	1
-459	val_459	2008-04-08	1
-459	val_459	2008-04-08	1
-150	val_150	2008-04-08	1
-150	val_150	2008-04-08	1
-176	val_176	2008-04-08	1
-176	val_176	2008-04-08	1
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
 176	val_176	2008-04-08	1
 176	val_176	2008-04-08	1
 183	val_183	2008-04-08	1
-183	val_183	2008-04-08	1
-187	val_187	2008-04-08	1
-187	val_187	2008-04-08	1
+187	val_187	2008-04-08	0
 187	val_187	2008-04-08	1
 187	val_187	2008-04-08	1
+19	val_19	2008-04-08	1
+190	val_190	2008-04-08	0
+194	val_194	2008-04-08	1
 202	val_202	2008-04-08	1
-202	val_202	2008-04-08	1
-228	val_228	2008-04-08	1
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
 228	val_228	2008-04-08	1
 235	val_235	2008-04-08	1
-235	val_235	2008-04-08	1
-257	val_257	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
 257	val_257	2008-04-08	1
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
+260	val_260	2008-04-08	1
+275	val_275	2008-04-08	0
+282	val_282	2008-04-08	0
+282	val_282	2008-04-08	1
+286	val_286	2008-04-08	1
+305	val_305	2008-04-08	0
+309	val_309	2008-04-08	0
 309	val_309	2008-04-08	1
-309	val_309	2008-04-08	1
-374	val_374	2008-04-08	1
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+316	val_316	2008-04-08	0
+323	val_323	2008-04-08	1
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	0
+327	val_327	2008-04-08	1
+33	val_33	2008-04-08	0
+338	val_338	2008-04-08	0
+341	val_341	2008-04-08	0
+345	val_345	2008-04-08	1
+356	val_356	2008-04-08	1
+367	val_367	2008-04-08	0
+367	val_367	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
 374	val_374	2008-04-08	1
+378	val_378	2008-04-08	1
+389	val_389	2008-04-08	0
 392	val_392	2008-04-08	1
-392	val_392	2008-04-08	1
-396	val_396	2008-04-08	1
-396	val_396	2008-04-08	1
+396	val_396	2008-04-08	0
 396	val_396	2008-04-08	1
 396	val_396	2008-04-08	1
+4	val_4	2008-04-08	0
+400	val_400	2008-04-08	0
+404	val_404	2008-04-08	0
+404	val_404	2008-04-08	0
+411	val_411	2008-04-08	1
+419	val_419	2008-04-08	0
+437	val_437	2008-04-08	0
+44	val_44	2008-04-08	1
+444	val_444	2008-04-08	1
+448	val_448	2008-04-08	0
+455	val_455	2008-04-08	1
+459	val_459	2008-04-08	0
+459	val_459	2008-04-08	1
+462	val_462	2008-04-08	1
+462	val_462	2008-04-08	1
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	1
+477	val_477	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+480	val_480	2008-04-08	0
+484	val_484	2008-04-08	0
 491	val_491	2008-04-08	1
-491	val_491	2008-04-08	1
+495	val_495	2008-04-08	0
+51	val_51	2008-04-08	1
+51	val_51	2008-04-08	1
+66	val_66	2008-04-08	1
+77	val_77	2008-04-08	0
+8	val_8	2008-04-08	0
+80	val_80	2008-04-08	0
+84	val_84	2008-04-08	0
+84	val_84	2008-04-08	0
+95	val_95	2008-04-08	0
+95	val_95	2008-04-08	1
 PREHOOK: query: drop table srcbucket_mapjoin_n8
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@srcbucket_mapjoin_n8
@@ -1986,11 +1678,11 @@ POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partit
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@srcbucket_mapjoin_n8
-PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8
+PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
 PREHOOK: type: QUERY
 PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8
+POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 STAGE DEPENDENCIES:
@@ -2011,29 +1703,28 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: srcbucket_mapjoin_n8__temp_table_for_load_data__
-                  Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
                   Select Operator
                     expressions: key (type: int), value (type: string), ds (type: string), hr (type: int)
                     outputColumnNames: _col0, _col1, _col2, _col3
-                    Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE
+                    Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
-                      key expressions: _col0 (type: int)
-                      sort order: +
-                      Map-reduce partition columns: _col0 (type: int)
-                      Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int)
+                      key expressions: _col2 (type: string), _col3 (type: int), _bucket_number (type: string), _col0 (type: int)
+                      sort order: ++++
+                      Map-reduce partition columns: _col2 (type: string), _col3 (type: int)
+                      value expressions: _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: vectorized, llap
             Reduce Operator Tree:
               Select Operator
-                expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int)
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE
+                expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._bucket_number (type: string)
+                outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number
                 File Output Operator
                   compressed: false
-                  Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE
+                  Dp Sort State: PARTITION_BUCKET_SORTED
+                  Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2060,11 +1751,11 @@ STAGE PLANS:
     Stats Work
       Basic Stats Work:
 
-PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8
+PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
 PREHOOK: type: QUERY
 PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 PREHOOK: Output: default@srcbucket_mapjoin_n8
-POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8
+POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__
 POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
@@ -2085,224 +1776,208 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0
 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
 #### A masked pattern was here ####
-8	val_8	2008-04-08	0
-8	val_8	2008-04-08	0
-8	val_8	2008-04-08	0
-8	val_8	2008-04-08	0
-33	val_33	2008-04-08	0
-33	val_33	2008-04-08	0
-33	val_33	2008-04-08	0
-33	val_33	2008-04-08	0
-103	val_103	2008-04-08	0
-103	val_103	2008-04-08	0
-103	val_103	2008-04-08	0
-103	val_103	2008-04-08	0
-125	val_125	2008-04-08	0
-125	val_125	2008-04-08	0
-125	val_125	2008-04-08	0
-125	val_125	2008-04-08	0
-275	val_275	2008-04-08	0
-275	val_275	2008-04-08	0
-275	val_275	2008-04-08	0
-275	val_275	2008-04-08	0
-305	val_305	2008-04-08	0
-305	val_305	2008-04-08	0
-305	val_305	2008-04-08	0
-305	val_305	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-466	val_466	2008-04-08	0
-484	val_484	2008-04-08	0
-484	val_484	2008-04-08	0
-484	val_484	2008-04-08	0
-484	val_484	2008-04-08	0
-0	val_0	2008-04-08	0
 0	val_0	2008-04-08	0
 0	val_0	2008-04-08	0
-0	val_0	2008-04-08	0
-4	val_4	2008-04-08	0
-4	val_4	2008-04-08	0
-4	val_4	2008-04-08	0
-4	val_4	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-26	val_26	2008-04-08	0
-77	val_77	2008-04-08	0
-77	val_77	2008-04-08	0
-77	val_77	2008-04-08	0
-77	val_77	2008-04-08	0
-118	val_118	2008-04-08	0
-118	val_118	2008-04-08	0
-118	val_118	2008-04-08	0
-118	val_118	2008-04-08	0
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+0	val_0	2008-04-08	1
+103	val_103	2008-04-08	0
+103	val_103	2008-04-08	0
+103	val_103	2008-04-08	1
+103	val_103	2008-04-08	1
+11	val_11	2008-04-08	1
+11	val_11	2008-04-08	1
+114	val_114	2008-04-08	1
+114	val_114	2008-04-08	1
 118	val_118	2008-04-08	0
 118	val_118	2008-04-08	0
 118	val_118	2008-04-08	0
 118	val_118	2008-04-08	0
+125	val_125	2008-04-08	0
+125	val_125	2008-04-08	0
+125	val_125	2008-04-08	1
+125	val_125	2008-04-08	1
+129	val_129	2008-04-08	0
+129	val_129	2008-04-08	0
+129	val_129	2008-04-08	1
+129	val_129	2008-04-08	1
+136	val_136	2008-04-08	1
+136	val_136	2008-04-08	1
+143	val_143	2008-04-08	0
+143	val_143	2008-04-08	0
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+15	val_15	2008-04-08	1
+150	val_150	2008-04-08	1
+150	val_150	2008-04-08	1
 158	val_158	2008-04-08	0
 158	val_158	2008-04-08	0
-158	val_158	2008-04-08	0
-158	val_158	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
-316	val_316	2008-04-08	0
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
+165	val_165	2008-04-08	1
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	0
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+169	val_169	2008-04-08	1
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
+172	val_172	2008-04-08	0
+176	val_176	2008-04-08	1
+176	val_176	2008-04-08	1
+176	val_176	2008-04-08	1
+176	val_176	2008-04-08	1
+183	val_183	2008-04-08	1
+183	val_183	2008-04-08	1
+187	val_187	2008-04-08	0
+187	val_187	2008-04-08	0
+187	val_187	2008-04-08	1
+187	val_187	2008-04-08	1
+187	val_187	2008-04-08	1
+187	val_187	2008-04-08	1
+19	val_19	2008-04-08	1
+19	val_19	2008-04-08	1
+190	val_190	2008-04-08	0
+190	val_190	2008-04-08	0
+194	val_194	2008-04-08	1
+194	val_194	2008-04-08	1
+202	val_202	2008-04-08	1
+202	val_202	2008-04-08	1
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
+213	val_213	2008-04-08	0
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
+217	val_217	2008-04-08	1
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
+224	val_224	2008-04-08	0
+228	val_228	2008-04-08	1
+228	val_228	2008-04-08	1
+235	val_235	2008-04-08	1
+235	val_235	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
+239	val_239	2008-04-08	1
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
+242	val_242	2008-04-08	0
+257	val_257	2008-04-08	1
+257	val_257	2008-04-08	1
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
+26	val_26	2008-04-08	0
+260	val_260	2008-04-08	1
+260	val_260	2008-04-08	1
+275	val_275	2008-04-08	0
+275	val_275	2008-04-08	0
+282	val_282	2008-04-08	0
+282	val_282	2008-04-08	0
+282	val_282	2008-04-08	1
+282	val_282	2008-04-08	1
+286	val_286	2008-04-08	1
+286	val_286	2008-04-08	1
+305	val_305	2008-04-08	0
+305	val_305	2008-04-08	0
+309	val_309	2008-04-08	0
+309	val_309	2008-04-08	0
+309	val_309	2008-04-08	1
+309	val_309	2008-04-08	1
 316	val_316	2008-04-08	0
 316	val_316	2008-04-08	0
 316	val_316	2008-04-08	0
 316	val_316	2008-04-08	0
 316	val_316	2008-04-08	0
 316	val_316	2008-04-08	0
+323	val_323	2008-04-08	1
+323	val_323	2008-04-08	1
 327	val_327	2008-04-08	0
 327	val_327	2008-04-08	0
 327	val_327	2008-04-08	0
 327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-327	val_327	2008-04-08	0
-338	val_338	2008-04-08	0
-338	val_338	2008-04-08	0
+327	val_327	2008-04-08	1
+327	val_327	2008-04-08	1
+33	val_33	2008-04-08	0
+33	val_33	2008-04-08	0
 338	val_338	2008-04-08	0
 338	val_338	2008-04-08	0
+341	val_341	2008-04-08	0
+341	val_341	2008-04-08	0
+345	val_345	2008-04-08	1
+345	val_345	2008-04-08	1
+356	val_356	2008-04-08	1
+356	val_356	2008-04-08	1
+367	val_367	2008-04-08	0
+367	val_367	2008-04-08	0
+367	val_367	2008-04-08	1
+367	val_367	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+37	val_37	2008-04-08	1
+374	val_374	2008-04-08	1
+374	val_374	2008-04-08	1
+378	val_378	2008-04-08	1
+378	val_378	2008-04-08	1
 389	val_389	2008-04-08	0
 389	val_389	2008-04-08	0
-389	val_389	2008-04-08	0
-389	val_389	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
-404	val_404	2008-04-08	0
+392	val_392	2008-04-08	1
+392	val_392	2008-04-08	1
+396	val_396	2008-04-08	0
+396	val_396	2008-04-08	0
+396	val_396	2008-04-08	1
+396	val_396	2008-04-08	1
+396	val_396	2008-04-08	1
+396	val_396	2008-04-08	1
+4	val_4	2008-04-08	0
+4	val_4	2008-04-08	0
+400	val_400	2008-04-08	0
+400	val_400	2008-04-08	0
 404	val_404	2008-04-08	0
 404	val_404	2008-04-08	0
 404	val_404	2008-04-08	0
 404	val_404	2008-04-08	0
+411	val_411	2008-04-08	1
+411	val_411	2008-04-08	1
 419	val_419	2008-04-08	0
 419	val_419	2008-04-08	0
-419	val_419	2008-04-08	0
-419	val_419	2008-04-08	0
-80	val_80	2008-04-08	0
-80	val_80	2008-04-08	0
-80	val_80	2008-04-08	0
-80	val_80	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-84	val_84	2008-04-08	0
-95	val_95	2008-04-08	0
-95	val_95	2008-04-08	0
-95	val_95	2008-04-08	0
-95	val_95	2008-04-08	0
-129	val_129	2008-04-08	0
-129	val_129	2008-04-08	0
-129	val_129	2008-04-08	0
-129	val_129	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-169	val_169	2008-04-08	0
-282	val_282	2008-04-08	0
-282	val_282	2008-04-08	0
-282	val_282	2008-04-08	0
-282	val_282	2008-04-08	0
-367	val_367	2008-04-08	0
-367	val_367	2008-04-08	0
-367	val_367	2008-04-08	0
-367	val_367	2008-04-08	0
-143	val_143	2008-04-08	0
-143	val_143	2008-04-08	0
-143	val_143	2008-04-08	0
-143	val_143	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-172	val_172	2008-04-08	0
-190	val_190	2008-04-08	0
-190	val_190	2008-04-08	0
-190	val_190	2008-04-08	0
-190	val_190	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-242	val_242	2008-04-08	0
-400	val_400	2008-04-08	0
-400	val_400	2008-04-08	0
-400	val_400	2008-04-08	0
-400	val_400	2008-04-08	0
-437	val_437	2008-04-08	0
 437	val_437	2008-04-08	0
 437	val_437	2008-04-08	0
-437	val_437	2008-04-08	0
-448	val_448	2008-04-08	0
-448	val_448	2008-04-08	0
+44	val_44	2008-04-08	1
+44	val_44	2008-04-08	1
+444	val_444	2008-04-08	1
+444	val_444	2008-04-08	1
 448	val_448	2008-04-08	0
 448	val_448	2008-04-08	0
+455	val_455	2008-04-08	1
+455	val_455	2008-04-08	1
 459	val_459	2008-04-08	0
 459	val_459	2008-04-08	0
-459	val_459	2008-04-08	0
-459	val_459	2008-04-08	0
-187	val_187	2008-04-08	0
-187	val_187	2008-04-08	0
-187	val_187	2008-04-08	0
-187	val_187	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-213	val_213	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-224	val_224	2008-04-08	0
-309	val_309	2008-04-08	0
-309	val_309	2008-04-08	0
-309	val_309	2008-04-08	0
-309	val_309	2008-04-08	0
-341	val_341	2008-04-08	0
-341	val_341	2008-04-08	0
-341	val_341	2008-04-08	0
-341	val_341	2008-04-08	0
-396	val_396	2008-04-08	0
-396	val_396	2008-04-08	0
-396	val_396	2008-04-08	0
-396	val_396	2008-04-08	0
-477	val_477	2008-04-08	0
-477	val_477	2008-04-08	0
+459	val_459	2008-04-08	1
+459	val_459	2008-04-08	1
+462	val_462	2008-04-08	1
+462	val_462	2008-04-08	1
+462	val_462	2008-04-08	1
+462	val_462	2008-04-08	1
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	0
+466	val_466	2008-04-08	1
+466	val_466	2008-04-08	1
 477	val_477	2008-04-08	0
 477	val_477	2008-04-08	0
 480	val_480	2008-04-08	0
@@ -2311,72 +1986,150 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1
 480	val_480	2008-04-08	0
 480	val_480	2008-04-08	0
 480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-480	val_480	2008-04-08	0
-495	val_495	2008-04-08	0
-495	val_495	2008-04-08	0
+484	val_484	2008-04-08	0
+484	val_484	2008-04-08	0
+491	val_491	2008-04-08	1
+491	val_491	2008-04-08	1
 495	val_495	2008-04-08	0
 495	val_495	2008-04-08	0
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-37	val_37	2008-04-08	1
-51	val_51	2008-04-08	1
 51	val_51	2008-04-08	1
 51	val_51	2008-04-08	1
 51	val_51	2008-04-08	1
 51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-51	val_51	2008-04-08	1
-103	val_103	2008-04-08	1
-103	val_103	2008-04-08	1
-103	val_103	2008-04-08	1
-103	val_103	2008-04-08	1
-125	val_125	2008-04-08	1
-125	val_125	2008-04-08	1
-125	val_125	2008-04-08	1
-125	val_125	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-165	val_165	2008-04-08	1
-194	val_194	2008-04-08	1
-194	val_194	2008-04-08	1
-194	val_194	2008-04-08	1
-194	val_194	2008-04-08	1
-260	val_260	2008-04-08	1
-260	val_260	2008-04-08	1
-260	val_260	2008-04-08	1
-260	val_260	2008-04-08	1
-323	val_323	2008-04-08	1
-323	val_323	2008-04-08	1
-323	val_323	2008-04-08	1
-323	val_323	2008-04-08	1
-411	val_411	2008-04-08	1
-411	val_411	2008-04-08	1
-411	val_411	2008-04-08	1
-411	val_411	2008-04-08	1
-455	val_455	2008-04-08	1
-455	val_455	2008-04-08	1
-455	val_455	2008-04-08	1
-455	val_455	2008-04-08	1
-466	val_466	2008-04-08	1
-466	val_466	2008-04-08	1
-466	val_

<TRUNCATED>

[5/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
index 5fd1bf6..40dc5e9 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
@@ -515,20 +515,19 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _bucket_number (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string)
-                        Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
+                        value expressions: 'foo' (type: string), 'bar' (type: string)
             Execution mode: llap
             LLAP IO: may be used (ACID table)
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
               Select Operator
-                expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string)
+                expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number
-                Statistics: Num rows: 5 Data size: 2220 Basic stats: COMPLETE Column stats: PARTIAL
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 5 Data size: 2220 Basic stats: COMPLETE Column stats: PARTIAL
+                  Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1286,20 +1285,19 @@ STAGE PLANS:
                         key expressions: '2008-04-08' (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: ++++
                         Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int)
-                        Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
+                        value expressions: 'foo' (type: string), 'bar' (type: string)
             Execution mode: llap
             LLAP IO: may be used (ACID table)
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
               Select Operator
-                expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string)
+                expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL
+                  Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1431,7 +1429,6 @@ STAGE PLANS:
                         key expressions: _col1 (type: string), _col2 (type: int), _bucket_number (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: ++++
                         Map-reduce partition columns: _col1 (type: string), _col2 (type: int)
-                        Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
             Execution mode: llap
             LLAP IO: may be used (ACID table)
         Reducer 2 
@@ -1440,11 +1437,10 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), KEY._col1 (type: string), KEY._col2 (type: int), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _bucket_number
-                Statistics: Num rows: 5 Data size: 2240 Basic stats: COMPLETE Column stats: PARTIAL
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 5 Data size: 2240 Basic stats: COMPLETE Column stats: PARTIAL
+                  Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1636,7 +1632,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: ++++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: int)
-                        Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
                         value expressions: _col1 (type: string), 'bar' (type: string)
             Execution mode: llap
             LLAP IO: may be used (ACID table)
@@ -1646,11 +1641,10 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL
+                  Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1745,7 +1739,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
                         sort order: ++++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: int)
-                        Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
                         value expressions: _col1 (type: string), 'bar' (type: string)
             Execution mode: llap
             LLAP IO: may be used (ACID table)
@@ -1755,11 +1748,10 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL
+                  Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out
index 9c7babb..916db93 100644
--- a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out
+++ b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out
@@ -299,7 +299,7 @@ STAGE PLANS:
               serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip}
               serialization.format 1
               serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
-              totalSize 545
+              totalSize 539
 #### A masked pattern was here ####
             serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
           
@@ -391,7 +391,7 @@ STAGE PLANS:
               serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip}
               serialization.format 1
               serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
-              totalSize 595
+              totalSize 594
 #### A masked pattern was here ####
             serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
           
@@ -690,7 +690,7 @@ STAGE PLANS:
               serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip}
               serialization.format 1
               serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
-              totalSize 545
+              totalSize 539
 #### A masked pattern was here ####
             serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
           
@@ -782,7 +782,7 @@ STAGE PLANS:
               serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip}
               serialization.format 1
               serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
-              totalSize 595
+              totalSize 594
 #### A masked pattern was here ####
             serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
           

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/lineage3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out
index cf38816..9bec309 100644
--- a/ql/src/test/results/clientpositive/llap/lineage3.q.out
+++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out
@@ -372,7 +372,7 @@ PREHOOK: query: create table src_dp1 (f string, w string, m int)
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@src_dp1
-Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[62][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
 PREHOOK: query: from src_dp, src_dp1
 insert into dest_dp1 partition (year) select first, word, year
 insert into dest_dp2 partition (y, m) select first, word, year, month

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out
index 7e6e88b..18dca49 100644
--- a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out
@@ -1916,7 +1916,7 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: oft
-                  Statistics: Num rows: 12288 Data size: 1884148 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 12288 Data size: 13243096 Basic stats: COMPLETE Column stats: PARTIAL
                   TableScan Vectorization:
                       native: true
                       vectorizationSchemaColumns: [0:csmallint:smallint, 1:cint:int, 2:cbigint:bigint, 3:cfloat:float, 4:cdouble:double, 5:cstring1:string, 6:cchar1:char(255), 7:cvchar1:varchar(255), 8:cboolean1:boolean, 9:cboolean2:boolean, 10:ctinyint:tinyint, 11:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
@@ -1939,7 +1939,7 @@ STAGE PLANS:
                     outputColumnNames: _col1, _col6, _col7, _col10
                     input vertices:
                       1 Map 2
-                    Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL
                     Select Operator
                       expressions: _col10 (type: tinyint), _col1 (type: int), _col6 (type: char(255)), _col7 (type: varchar(255))
                       outputColumnNames: _col0, _col1, _col2, _col3
@@ -1947,13 +1947,13 @@ STAGE PLANS:
                           className: VectorSelectOperator
                           native: true
                           projectedOutputColumnNums: [10, 1, 6, 7]
-                      Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL
                       File Output Operator
                         compressed: false
                         File Sink Vectorization:
                             className: VectorFileSinkOperator
                             native: false
-                        Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL
                         table:
                             input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                             output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/llap_smb.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_smb.q.out b/ql/src/test/results/clientpositive/llap/llap_smb.q.out
index cdc10f8..5f5a747 100644
--- a/ql/src/test/results/clientpositive/llap/llap_smb.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_smb.q.out
@@ -269,10 +269,10 @@ STAGE PLANS:
                 TableScan
                   alias: a
                   filterExpr: id is not null (type: boolean)
-                  Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                   Filter Operator
                     predicate: id is not null (type: boolean)
-                    Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                     Merge Join Operator
                       condition map:
                            Inner Join 0 to 1
@@ -280,18 +280,18 @@ STAGE PLANS:
                         0 id (type: bigint)
                         1 id (type: bigint)
                       outputColumnNames: _col2, _col3
-                      Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                       Group By Operator
                         aggregations: count()
                         keys: _col2 (type: int), _col3 (type: smallint)
                         mode: hash
                         outputColumnNames: _col0, _col1, _col2
-                        Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                         Reduce Output Operator
                           key expressions: _col0 (type: int), _col1 (type: smallint)
                           sort order: ++
                           Map-reduce partition columns: _col0 (type: int), _col1 (type: smallint)
-                          Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                           value expressions: _col2 (type: bigint)
             Execution mode: llap
         Reducer 2 
@@ -302,10 +302,10 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int), KEY._col1 (type: smallint)
                 mode: mergepartial
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                 File Output Operator
                   compressed: false
-                  Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/llap_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_stats.q.out b/ql/src/test/results/clientpositive/llap/llap_stats.q.out
index 5bba6b0..89c0c57 100644
--- a/ql/src/test/results/clientpositive/llap/llap_stats.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_stats.q.out
@@ -153,11 +153,11 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: llap_stats
-                  Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: PARTIAL
+                  Statistics: Num rows: 10 Data size: 196 Basic stats: COMPLETE Column stats: PARTIAL
                   Select Operator
                     expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int)
                     outputColumnNames: ctinyint, csmallint, cint
-                    Statistics: Num rows: 10 Data size: 120 Basic stats: COMPLETE Column stats: PARTIAL
+                    Statistics: Num rows: 10 Data size: 196 Basic stats: COMPLETE Column stats: PARTIAL
                     Group By Operator
                       aggregations: compute_stats(ctinyint, 'hll'), compute_stats(csmallint, 'hll')
                       keys: cint (type: int)


[7/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Posted by vg...@apache.org.
HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5eebbdf7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5eebbdf7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5eebbdf7

Branch: refs/heads/master
Commit: 5eebbdf7c5750b31e1c43fe576fc0ab728bce05c
Parents: 0d39a81
Author: Vineet Garg <vg...@apache.org>
Authored: Sun Oct 21 12:45:04 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Sun Oct 21 12:45:04 2018 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   10 +
 .../insert_into_dynamic_partitions.q.out        |  126 +-
 .../insert_overwrite_dynamic_partitions.q.out   |  126 +-
 .../clientpositive/orc_format_part.q.out        |   14 +-
 .../orc_nonstd_partitions_loc.q.out             |    8 +-
 .../clientpositive/parquet_format_part.q.out    |   14 +-
 .../parquet_nonstd_partitions_loc.q.out         |    8 +-
 .../clientpositive/rcfile_format_part.q.out     |   14 +-
 .../rcfile_nonstd_partitions_loc.q.out          |    6 +-
 .../jdbc/TestTriggersTezSessionPoolManager.java |    4 +
 .../apache/hadoop/hive/ql/exec/MemoryInfo.java  |  114 +
 .../hadoop/hive/ql/optimizer/Optimizer.java     |    7 +-
 .../optimizer/SortedDynPartitionOptimizer.java  |  112 +-
 .../physical/LlapClusterStateForCompile.java    |   12 +
 .../hadoop/hive/ql/parse/TezCompiler.java       |   24 +
 .../test/queries/clientpositive/dp_counter_mm.q |    1 +
 .../queries/clientpositive/dp_counter_non_mm.q  |    1 +
 .../dynpart_sort_opt_vectorization.q            |   25 +-
 .../clientpositive/dynpart_sort_optimization.q  |   63 +-
 .../clientpositive/dynpart_sort_optimization2.q |    9 +-
 .../dynpart_sort_optimization_acid.q            |   12 +-
 .../dynpart_sort_optimization_acid2.q           |    2 +-
 .../clientpositive/load_data_using_job.q        |    2 +
 .../queries/clientpositive/tez_input_counters.q |    1 +
 .../vector_partitioned_date_time.q              |    3 +-
 .../clientpositive/acid_table_stats.q.out       |   16 +-
 .../dynpart_sort_optimization_acid2.q.out       |   88 +-
 .../llap/auto_sortmerge_join_16.q.out           |   32 +-
 .../llap/dynpart_sort_opt_vectorization.q.out   |   44 +-
 .../llap/dynpart_sort_optimization.q.out        |  489 ++-
 .../llap/dynpart_sort_optimization2.q.out       |    8 +-
 .../llap/dynpart_sort_optimization_acid.q.out   |   26 +-
 .../extrapolate_part_stats_partial_ndv.q.out    |    8 +-
 .../results/clientpositive/llap/lineage3.q.out  |    2 +-
 .../clientpositive/llap/llap_partitioned.q.out  |    8 +-
 .../results/clientpositive/llap/llap_smb.q.out  |   14 +-
 .../clientpositive/llap/llap_stats.q.out        |    4 +-
 .../llap/load_data_using_job.q.out              | 3624 +++++++++---------
 .../clientpositive/llap/load_dyn_part5.q.out    |   60 +-
 .../llap/materialized_view_partitioned_3.q.out  |    2 -
 .../results/clientpositive/llap/mm_dp.q.out     |    4 +-
 .../clientpositive/llap/orc_analyze.q.out       |    6 +-
 .../clientpositive/llap/orc_merge1.q.out        |  187 +-
 .../clientpositive/llap/orc_merge10.q.out       |  338 +-
 .../clientpositive/llap/orc_merge2.q.out        |   60 +-
 .../clientpositive/llap/orc_merge7.q.out        |  108 +-
 .../clientpositive/llap/orc_merge_diff_fs.q.out |  176 +-
 .../llap/orc_merge_incompat2.q.out              |   54 +-
 .../clientpositive/llap/subquery_notin.q.out    |   24 +-
 .../results/clientpositive/llap/tez_dml.q.out   |   60 +-
 .../llap/vector_count_distinct.q.out            |   18 +-
 .../llap/vector_partitioned_date_time.q.out     | 2730 ++++++-------
 52 files changed, 4726 insertions(+), 4182 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index ab86200..bcf1e9e 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2230,6 +2230,16 @@ public class HiveConf extends Configuration {
         "When enabled dynamic partitioning column will be globally sorted.\n" +
         "This way we can keep only one record writer open for each partition value\n" +
         "in the reducer thereby reducing the memory pressure on reducers."),
+    HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD("hive.optimize.sort.dynamic.partition.threshold", 0,
+                                "When enabled dynamic partitioning column will be globally sorted.\n" +
+                                    "This way we can keep only one record writer open for each partition value\n" +
+                                    "in the reducer thereby reducing the memory pressure on reducers.\n" +
+                                    "This config has following possible values: \n" +
+                                    "\t-1 - This completely disables the optimization. \n" +
+                                    "\t1 - This always enable the optimization. \n" +
+                                    "\t0 - This makes the optimization a cost based decision. \n" +
+                                    "Setting it to any other positive integer will make Hive use this as threshold for number of writers."),
+
 
     HIVESAMPLINGFORORDERBY("hive.optimize.sampling.orderby", false, "Uses sampling on order-by clause for parallel execution."),
     HIVESAMPLINGNUMBERFORORDERBY("hive.optimize.sampling.orderby.number", 1000, "Total number of samples to be obtained."),

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
index b42f966..80dbbee 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
@@ -84,7 +84,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
-  Stage-2 depends on stages: Stage-0
+  Stage-2 depends on stages: Stage-0, Stage-3
+  Stage-3 depends on stages: Stage-1
 
 STAGE PLANS:
   Stage: Stage-1
@@ -107,13 +108,12 @@ STAGE PLANS:
                   outputColumnNames: _col0, _col1
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
-                    key expressions: _col1 (type: string), _bucket_number (type: string)
-                    null sort order: aa
-                    sort order: ++
-                    Map-reduce partition columns: _col1 (type: string)
+                    null sort order: 
+                    sort order: 
+                    Map-reduce partition columns: _col0 (type: int)
                     Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
                     tag: -1
-                    value expressions: _col0 (type: int)
+                    value expressions: _col0 (type: int), _col1 (type: string)
                     auto parallelism: false
       Path -> Alias:
 #### A masked pattern was here ####
@@ -159,16 +159,15 @@ STAGE PLANS:
       Needs Tagging: false
       Reduce Operator Tree:
         Select Operator
-          expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string)
-          outputColumnNames: _col0, _col1, _bucket_number
-          Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+          expressions: VALUE._col0 (type: int), VALUE._col1 (type: string)
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
           File Output Operator
             compressed: false
             GlobalTableId: 1
             directory: ### BLOBSTORE_STAGING_PATH ###
-            Dp Sort State: PARTITION_BUCKET_SORTED
             NumFilesPerFileSink: 1
-            Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
             Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
             table:
                 input format: org.apache.hadoop.mapred.TextInputFormat
@@ -195,6 +194,34 @@ STAGE PLANS:
             TotalFiles: 1
             GatherStats: true
             MultiFileSpray: false
+          Select Operator
+            expressions: _col0 (type: int), _col1 (type: string)
+            outputColumnNames: id, key
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+            Group By Operator
+              aggregations: compute_stats(id, 'hll')
+              keys: key (type: string)
+              mode: hash
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+#### A masked pattern was here ####
+                NumFilesPerFileSink: 1
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    properties:
+                      column.name.delimiter ,
+                      columns _col0,_col1
+                      columns.types string,struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>
+                      escape.delim \
+                      serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                TotalFiles: 1
+                GatherStats: false
+                MultiFileSpray: false
 
   Stage: Stage-0
     Move Operator
@@ -236,6 +263,83 @@ STAGE PLANS:
           Table: default.table1
           Is Table Level Stats: false
 
+  Stage: Stage-3
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            GatherStats: false
+            Reduce Output Operator
+              key expressions: _col0 (type: string)
+              null sort order: a
+              sort order: +
+              Map-reduce partition columns: _col0 (type: string)
+              Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE
+              tag: -1
+              value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>)
+              auto parallelism: false
+      Execution mode: vectorized
+      Path -> Alias:
+#### A masked pattern was here ####
+      Path -> Partition:
+#### A masked pattern was here ####
+          Partition
+            base file name: -mr-10002
+            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+            properties:
+              column.name.delimiter ,
+              columns _col0,_col1
+              columns.types string,struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>
+              escape.delim \
+              serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+            serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          
+              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+              properties:
+                column.name.delimiter ,
+                columns _col0,_col1
+                columns.types string,struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>
+                escape.delim \
+                serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+              serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Truncated Path -> Alias:
+#### A masked pattern was here ####
+      Needs Tagging: false
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: compute_stats(VALUE._col0)
+          keys: KEY._col0 (type: string)
+          mode: mergepartial
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE
+          Select Operator
+            expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+#### A masked pattern was here ####
+              NumFilesPerFileSink: 1
+              Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  properties:
+                    columns _col0,_col1
+                    columns.types struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>:string
+                    escape.delim \
+                    hive.serialization.extend.additional.nesting.levels true
+                    serialization.escape.crlf true
+                    serialization.format 1
+                    serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              TotalFiles: 1
+              GatherStats: false
+              MultiFileSpray: false
+
 PREHOOK: query: DROP TABLE table1
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@table1

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
index 13d64fb..a8cdc8f 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
@@ -102,7 +102,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
-  Stage-2 depends on stages: Stage-0
+  Stage-2 depends on stages: Stage-0, Stage-3
+  Stage-3 depends on stages: Stage-1
 
 STAGE PLANS:
   Stage: Stage-1
@@ -125,13 +126,12 @@ STAGE PLANS:
                   outputColumnNames: _col0, _col1
                   Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
-                    key expressions: _col1 (type: string), _bucket_number (type: string)
-                    null sort order: aa
-                    sort order: ++
-                    Map-reduce partition columns: _col1 (type: string)
+                    null sort order: 
+                    sort order: 
+                    Map-reduce partition columns: _col0 (type: int)
                     Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
                     tag: -1
-                    value expressions: _col0 (type: int)
+                    value expressions: _col0 (type: int), _col1 (type: string)
                     auto parallelism: false
       Path -> Alias:
 #### A masked pattern was here ####
@@ -177,16 +177,15 @@ STAGE PLANS:
       Needs Tagging: false
       Reduce Operator Tree:
         Select Operator
-          expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string)
-          outputColumnNames: _col0, _col1, _bucket_number
-          Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+          expressions: VALUE._col0 (type: int), VALUE._col1 (type: string)
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
           File Output Operator
             compressed: false
             GlobalTableId: 1
             directory: ### BLOBSTORE_STAGING_PATH ###
-            Dp Sort State: PARTITION_BUCKET_SORTED
             NumFilesPerFileSink: 1
-            Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
             Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ###
             table:
                 input format: org.apache.hadoop.mapred.TextInputFormat
@@ -213,6 +212,34 @@ STAGE PLANS:
             TotalFiles: 1
             GatherStats: true
             MultiFileSpray: false
+          Select Operator
+            expressions: _col0 (type: int), _col1 (type: string)
+            outputColumnNames: id, key
+            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+            Group By Operator
+              aggregations: compute_stats(id, 'hll')
+              keys: key (type: string)
+              mode: hash
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE
+              File Output Operator
+                compressed: false
+                GlobalTableId: 0
+#### A masked pattern was here ####
+                NumFilesPerFileSink: 1
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    properties:
+                      column.name.delimiter ,
+                      columns _col0,_col1
+                      columns.types string,struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>
+                      escape.delim \
+                      serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                TotalFiles: 1
+                GatherStats: false
+                MultiFileSpray: false
 
   Stage: Stage-0
     Move Operator
@@ -254,6 +281,83 @@ STAGE PLANS:
           Table: default.table1
           Is Table Level Stats: false
 
+  Stage: Stage-3
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            GatherStats: false
+            Reduce Output Operator
+              key expressions: _col0 (type: string)
+              null sort order: a
+              sort order: +
+              Map-reduce partition columns: _col0 (type: string)
+              Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE
+              tag: -1
+              value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>)
+              auto parallelism: false
+      Execution mode: vectorized
+      Path -> Alias:
+#### A masked pattern was here ####
+      Path -> Partition:
+#### A masked pattern was here ####
+          Partition
+            base file name: -mr-10002
+            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+            properties:
+              column.name.delimiter ,
+              columns _col0,_col1
+              columns.types string,struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>
+              escape.delim \
+              serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+            serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          
+              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+              properties:
+                column.name.delimiter ,
+                columns _col0,_col1
+                columns.types string,struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>
+                escape.delim \
+                serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+              serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Truncated Path -> Alias:
+#### A masked pattern was here ####
+      Needs Tagging: false
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: compute_stats(VALUE._col0)
+          keys: KEY._col0 (type: string)
+          mode: mergepartial
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE
+          Select Operator
+            expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+#### A masked pattern was here ####
+              NumFilesPerFileSink: 1
+              Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE
+#### A masked pattern was here ####
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  properties:
+                    columns _col0,_col1
+                    columns.types struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>:string
+                    escape.delim \
+                    hive.serialization.extend.additional.nesting.levels true
+                    serialization.escape.crlf true
+                    serialization.format 1
+                    serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              TotalFiles: 1
+              GatherStats: false
+              MultiFileSpray: false
+
 PREHOOK: query: DROP TABLE table1
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@table1

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out
index 7b25613..826fae9 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out
@@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-200
+0
 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_events
@@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-50
+0
 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121
 PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_events
@@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-100
+0
 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id, event_name)
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
 WHERE SUBSTR(run_date,1,6)='201211'
@@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-300
+0
 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name)
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
@@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-350
+0
 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
@@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-400
+50
 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
@@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-350
+50

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out
index 1201ce2..bb63070 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out
@@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-200
+0
 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
 #### A masked pattern was here ####
 PREHOOK: type: ALTERTABLE_ADDPARTS
@@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-300
+100
 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201211'
@@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-400
+200
 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 #### A masked pattern was here ####
 PREHOOK: type: ALTERTABLE_ADDPARTS
@@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_events
 #### A masked pattern was here ####
-500
+300
 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name)
 SELECT * FROM src_events
 PREHOOK: type: QUERY

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
index 0931e3d..7758dc9 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out
@@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-200
+0
 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
 PREHOOK: type: QUERY
 PREHOOK: Input: default@parquet_events
@@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-50
+0
 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
 PREHOOK: type: QUERY
 PREHOOK: Input: default@parquet_events
@@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-100
+0
 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name)
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
 WHERE SUBSTR(run_date,1,6)='201211'
@@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-300
+0
 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name)
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
@@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-350
+0
 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
@@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-400
+50
 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
@@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-350
+50

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
index 15ae3d9..0ccd0e4 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out
@@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-200
+0
 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
 #### A masked pattern was here ####
 PREHOOK: type: ALTERTABLE_ADDPARTS
@@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-300
+100
 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201211'
@@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-400
+200
 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 #### A masked pattern was here ####
 PREHOOK: type: ALTERTABLE_ADDPARTS
@@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@parquet_events
 #### A masked pattern was here ####
-500
+300
 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name)
 SELECT * FROM src_events
 PREHOOK: type: QUERY

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out
index 24fc525..340791a 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out
@@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-200
+0
 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921
 PREHOOK: type: QUERY
 PREHOOK: Input: default@rcfile_events
@@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-50
+0
 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121
 PREHOOK: type: QUERY
 PREHOOK: Input: default@rcfile_events
@@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-100
+0
 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id, event_name)
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events
 WHERE SUBSTR(run_date,1,6)='201211'
@@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-300
+0
 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name)
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39
@@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-350
+0
 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
@@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-400
+50
 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'
@@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-350
+50

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out
index 6bcfe41..1608422 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out
@@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-200
+0
 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change')
 #### A masked pattern was here ####
 PREHOOK: type: ALTERTABLE_ADDPARTS
@@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-300
+100
 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change')
 SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events
 WHERE SUBSTR(run_date,1,6)='201211'
@@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@rcfile_events
 #### A masked pattern was here ####
-400
+200
 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change')
 #### A masked pattern was here ####
 PREHOOK: type: ALTERTABLE_ADDPARTS

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
index afea596..faab11a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java
@@ -169,6 +169,7 @@ public class TestTriggersTezSessionPoolManager extends AbstractJdbcTriggersTest
     List<String> cmds = getConfigs();
     cmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
     cmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
+    cmds.add("set hive.optimize.sort.dynamic.partition.threshold=-1");
     Expression expression = ExpressionFactory.fromString("CREATED_FILES > 5");
     Trigger trigger = new ExecutionTrigger("high_read_ops", expression, new Action(Action.Type.KILL_QUERY));
     setupTriggers(Lists.newArrayList(trigger));
@@ -193,6 +194,7 @@ public class TestTriggersTezSessionPoolManager extends AbstractJdbcTriggersTest
     cmds.add("create table src2 (key int) partitioned by (value string)");
     cmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
     cmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
+    cmds.add("set hive.optimize.sort.dynamic.partition.threshold=-1");
     // query will get cancelled before creating 57 partitions
     String query =
       "insert overwrite table src2 partition (value) select * from " + tableName + " where under_col < 100";
@@ -237,6 +239,7 @@ public class TestTriggersTezSessionPoolManager extends AbstractJdbcTriggersTest
     cmds.add("create table src3 (key int) partitioned by (value string)");
     cmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
     cmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
+    cmds.add("set hive.optimize.sort.dynamic.partition.threshold=-1");
     String query =
       "from " + tableName +
         " insert overwrite table src2 partition (value) select * where under_col < 100 " +
@@ -254,6 +257,7 @@ public class TestTriggersTezSessionPoolManager extends AbstractJdbcTriggersTest
     cmds.add("create table src2 (key int) partitioned by (value string)");
     cmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
     cmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
+    cmds.add("set hive.optimize.sort.dynamic.partition.threshold=-1");
     // query will get cancelled before creating 57 partitions
     String query =
       "insert overwrite table src2 partition (value) " +

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java
new file mode 100644
index 0000000..4c91c49
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.LlapUtil;
+import org.apache.hadoop.hive.ql.optimizer.physical.LlapClusterStateForCompile;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+
+/**
+ * Contains information about executor memory, various memory thresholds used for join conversions etc. based on
+ * execution engine.
+ **/
+
+public class MemoryInfo {
+
+  private Configuration conf;
+  private boolean isTez;
+  private boolean isLlap;
+  private long maxExecutorMemory;
+  private long mapJoinMemoryThreshold;
+  private long dynPartJoinMemoryThreshold;
+
+  public MemoryInfo(Configuration conf) {
+    this.isTez = "tez".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
+    this.isLlap = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE));
+    if (isLlap) {
+      LlapClusterStateForCompile llapInfo = LlapClusterStateForCompile.getClusterInfo(conf);
+      llapInfo.initClusterInfo();
+      if (llapInfo.hasClusterInfo()) {
+        this.maxExecutorMemory = llapInfo.getMemoryPerExecutor();
+      } else {
+        long memPerInstance =
+            HiveConf.getLongVar(conf, HiveConf.ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB) * 1024L * 1024L;
+        long numExecutors = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
+        this.maxExecutorMemory = memPerInstance / numExecutors;
+      }
+    } else {
+      if (isTez) {
+        float heapFraction = HiveConf.getFloatVar(conf, HiveConf.ConfVars.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION);
+        int containerSizeMb = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ?
+            HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) :
+            conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB);
+        // this can happen when config is explicitly set to "-1", in which case defaultValue also does not work
+        if (containerSizeMb < 0) {
+          containerSizeMb =  MRJobConfig.DEFAULT_MAP_MEMORY_MB;
+        }
+        this.maxExecutorMemory = (long) ((containerSizeMb * 1024L * 1024L) * heapFraction);
+      } else {
+        this.maxExecutorMemory =
+            conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB) * 1024L * 1024L;
+        // this can happen when config is explicitly set to "-1", in which case defaultValue also does not work
+        if (maxExecutorMemory < 0) {
+          maxExecutorMemory =  MRJobConfig.DEFAULT_MAP_MEMORY_MB * 1024L * 1024L;
+        }
+      }
+    }
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+  public void setConf(final Configuration conf) {
+    this.conf = conf;
+  }
+
+  public boolean isTez() {
+    return isTez;
+  }
+
+  public boolean isLlap() {
+    return isLlap;
+  }
+
+  public long getMaxExecutorMemory() {
+    return maxExecutorMemory;
+  }
+
+  public long getMapJoinMemoryThreshold() {
+    return mapJoinMemoryThreshold;
+  }
+
+  public long getDynPartJoinMemoryThreshold() {
+    return dynPartJoinMemoryThreshold;
+  }
+
+  @Override
+  public String toString() {
+    return "MEMORY INFO - { isTez: " + isTez() +
+        ", isLlap: " + isLlap() +
+        ", maxExecutorMemory: " + LlapUtil.humanReadableByteCount(getMaxExecutorMemory()) +
+        ", mapJoinMemoryThreshold: "+ LlapUtil.humanReadableByteCount(getMapJoinMemoryThreshold()) +
+        ", dynPartJoinMemoryThreshold: " + LlapUtil.humanReadableByteCount(getDynPartJoinMemoryThreshold()) +
+        " }";
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 71f7380..25e9cd0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -115,12 +115,7 @@ public class Optimizer {
       transformations.add(new ConstantPropagate());
     }
 
-    if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) &&
-        HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") &&
-        HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) &&
-        !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) {
-      transformations.add(new SortedDynPartitionOptimizer());
-    }
+
 
     transformations.add(new SortedDynPartitionTimeGranularityOptimizer());
 

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
index 314b8b4..4d9963a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
@@ -27,11 +27,13 @@ import java.util.Map;
 import java.util.Set;
 import java.util.Stack;
 
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.MemoryInfo;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
 import org.apache.hadoop.hive.ql.exec.OperatorUtils;
@@ -56,6 +58,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
 import org.apache.hadoop.hive.ql.parse.ParseUtils;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ColStatistics;
 import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -70,6 +73,7 @@ import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
 import org.apache.hadoop.hive.ql.plan.SelectDesc;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.orc.OrcConf;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,6 +90,7 @@ import com.google.common.collect.Sets;
 public class SortedDynPartitionOptimizer extends Transform {
 
   private static final String BUCKET_NUMBER_COL_NAME = "_bucket_number";
+
   @Override
   public ParseContext transform(ParseContext pCtx) throws SemanticException {
 
@@ -159,6 +164,12 @@ public class SortedDynPartitionOptimizer extends Transform {
         return null;
       }
 
+      DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx();
+      List<Integer> partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema());
+
+      if (!shouldDo(partitionPositions, fsParent)) {
+        return null;
+      }
       // if RS is inserted by enforce bucketing or sorting, we need to remove it
       // since ReduceSinkDeDuplication will not merge them to single RS.
       // RS inserted by enforce bucketing/sorting will have bucketing column in
@@ -175,13 +186,13 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       // unlink connection between FS and its parent
       fsParent = fsOp.getParentOperators().get(0);
+
       fsParent.getChildOperators().clear();
 
-      DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx();
-      int numBuckets = destTable.getNumBuckets();
 
       // if enforce bucketing/sorting is disabled numBuckets will not be set.
       // set the number of buckets here to ensure creation of empty buckets
+      int numBuckets = destTable.getNumBuckets();
       dpCtx.setNumBuckets(numBuckets);
 
       // Get the positions for partition, bucket and sort columns
@@ -206,7 +217,7 @@ public class SortedDynPartitionOptimizer extends Transform {
          * which extracts bucketId from it
          * see {@link org.apache.hadoop.hive.ql.udf.UDFToInteger#evaluate(RecordIdentifier)}*/
         ColumnInfo ci = fsParent.getSchema().getSignature().get(0);
-        if(!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) {
+        if (!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) {
           throw new IllegalStateException("expected 1st column to be ROW__ID but got wrong type: " + ci.toString());
         }
         //add a cast(ROW__ID as int) to wrap in UDFToInteger()
@@ -230,10 +241,15 @@ public class SortedDynPartitionOptimizer extends Transform {
         sortNullOrder.add(order == 1 ? 0 : 1); // for asc, nulls first; for desc, nulls last
       }
       LOG.debug("Got sort order");
-      for (int i : sortPositions) LOG.debug("sort position " + i);
-      for (int i : sortOrder) LOG.debug("sort order " + i);
-      for (int i : sortNullOrder) LOG.debug("sort null order " + i);
-      List<Integer> partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema());
+      for (int i : sortPositions) {
+        LOG.debug("sort position " + i);
+      }
+      for (int i : sortOrder) {
+        LOG.debug("sort order " + i);
+      }
+      for (int i : sortNullOrder) {
+        LOG.debug("sort null order " + i);
+      }
 
       // update file sink descriptor
       fsOp.getConf().setMultiFileSpray(false);
@@ -248,7 +264,8 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       // Create ReduceSink operator
       ReduceSinkOperator rsOp = getReduceSinkOp(partitionPositions, sortPositions, sortOrder, sortNullOrder,
-          allRSCols, bucketColumns, numBuckets, fsParent, fsOp.getConf().getWriteType());
+                                                  allRSCols, bucketColumns, numBuckets,
+                                                 fsParent, fsOp.getConf().getWriteType());
 
       List<ExprNodeDesc> descs = new ArrayList<ExprNodeDesc>(allRSCols.size());
       List<String> colNames = new ArrayList<String>();
@@ -265,9 +282,11 @@ public class SortedDynPartitionOptimizer extends Transform {
       }
       RowSchema selRS = new RowSchema(fsParent.getSchema());
       if (!bucketColumns.isEmpty()) {
-        descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, ReduceField.KEY.toString()+"."+BUCKET_NUMBER_COL_NAME, null, false));
+        descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
+                                         ReduceField.KEY.toString()+"."+BUCKET_NUMBER_COL_NAME, null, false));
         colNames.add(BUCKET_NUMBER_COL_NAME);
-        ColumnInfo ci = new ColumnInfo(BUCKET_NUMBER_COL_NAME, TypeInfoFactory.stringTypeInfo, selRS.getSignature().get(0).getTabAlias(), true, true);
+        ColumnInfo ci = new ColumnInfo(BUCKET_NUMBER_COL_NAME, TypeInfoFactory.stringTypeInfo,
+                                       selRS.getSignature().get(0).getTabAlias(), true, true);
         selRS.getSignature().add(ci);
         fsParent.getSchema().getSignature().add(ci);
       }
@@ -290,7 +309,7 @@ public class SortedDynPartitionOptimizer extends Transform {
       }
 
       // update partition column info in FS descriptor
-      fsOp.getConf().setPartitionCols( rsOp.getConf().getPartitionCols());
+      fsOp.getConf().setPartitionCols(rsOp.getConf().getPartitionCols());
 
       LOG.info("Inserted " + rsOp.getOperatorId() + " and " + selOp.getOperatorId()
           + " as parent of " + fsOp.getOperatorId() + " and child of " + fsParent.getOperatorId());
@@ -478,7 +497,7 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       String orderStr = "";
       for (Integer i : newSortOrder) {
-        if(i == 1) {
+        if (i == 1) {
           orderStr += "+";
         } else {
           orderStr += "-";
@@ -502,7 +521,7 @@ public class SortedDynPartitionOptimizer extends Transform {
 
       String nullOrderStr = "";
       for (Integer i : newSortNullOrder) {
-        if(i == 0) {
+        if (i == 0) {
           nullOrderStr += "a";
         } else {
           nullOrderStr += "z";
@@ -517,7 +536,7 @@ public class SortedDynPartitionOptimizer extends Transform {
       for (Integer idx : keyColsPosInVal) {
         if (idx < 0) {
           ExprNodeDesc bucketNumColUDF = ExprNodeGenericFuncDesc.newInstance(
-            FunctionRegistry.getFunctionInfo("bucket_number").getGenericUDF(), new ArrayList<>());
+              FunctionRegistry.getFunctionInfo("bucket_number").getGenericUDF(), new ArrayList<>());
           keyCols.add(bucketNumColUDF);
           colExprMap.put(Utilities.ReduceField.KEY + "." +BUCKET_NUMBER_COL_NAME, bucketNumColUDF);
         } else {
@@ -598,7 +617,8 @@ public class SortedDynPartitionOptimizer extends Transform {
     }
 
     /**
-     * Get the sort positions for the sort columns
+     * Get the sort positions for the sort columns.
+     *
      * @param tabSortCols
      * @param tabCols
      * @return
@@ -620,7 +640,8 @@ public class SortedDynPartitionOptimizer extends Transform {
     }
 
     /**
-     * Get the sort order for the sort columns
+     * Get the sort order for the sort columns.
+     *
      * @param tabSortCols
      * @param tabCols
      * @return
@@ -652,6 +673,63 @@ public class SortedDynPartitionOptimizer extends Transform {
       return cols;
     }
 
-  }
+    // the idea is to estimate how many number of writers this insert can spun up.
+    // Writers are proportional to number of partitions being inserted i.e cardinality of the partition columns
+    //  if these writers are less than number of writers allowed within the memory pool (estimated) we go ahead with
+    //  adding extra RS
+    //  The way max number of writers allowed are computed based on
+    //  (executor/container memory) * (percentage of memory taken by orc)
+    //  and dividing that by max memory (stripe size) taken by a single writer.
+    private boolean shouldDo(List<Integer> partitionPos, Operator<? extends OperatorDesc> fsParent) {
+
+      int threshold = HiveConf.getIntVar(this.parseCtx.getConf(),
+                                         HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD);
+      long MAX_WRITERS = -1;
+
+      switch (threshold) {
+      case -1:
+        return false;
+      case 0:
+        break;
+      case 1:
+        return true;
+      default:
+        MAX_WRITERS = threshold;
+        break;
+      }
+
+      List<ColStatistics> colStats = fsParent.getStatistics().getColumnStats();
+      if (colStats == null || colStats.isEmpty()) {
+        return true;
+      }
+      long partCardinality = 1;
+
+      // compute cardinality for partition columns
+      for (Integer idx : partitionPos) {
+        ColumnInfo ci = fsParent.getSchema().getSignature().get(idx);
+        ColStatistics partStats = fsParent.getStatistics().getColumnStatisticsFromColName(ci.getInternalName());
+        if (partStats == null) {
+          // statistics for this partition are for some reason not available
+          return true;
+        }
+        partCardinality = partCardinality * partStats.getCountDistint();
+      }
+
+      if (MAX_WRITERS < 0) {
+        double orcMemPool = this.parseCtx.getConf().getDouble(OrcConf.MEMORY_POOL.getHiveConfName(),
+                                                              (Double) OrcConf.MEMORY_POOL.getDefaultValue());
+        long orcStripSize = this.parseCtx.getConf().getLong(OrcConf.STRIPE_SIZE.getHiveConfName(),
+                                                            (Long) OrcConf.STRIPE_SIZE.getDefaultValue());
+        MemoryInfo memoryInfo = new MemoryInfo(this.parseCtx.getConf());
+        LOG.debug("Memory info during SDPO opt: {}", memoryInfo);
+        long executorMem = memoryInfo.getMaxExecutorMemory();
+        MAX_WRITERS = (long) (executorMem * orcMemPool) / orcStripSize;
 
+      }
+      if (partCardinality <= MAX_WRITERS) {
+        return false;
+      }
+      return true;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
index 0d1990a..a15bdd5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java
@@ -44,6 +44,7 @@ public class LlapClusterStateForCompile {
   private volatile Long lastClusterUpdateNs;
   private volatile Integer noConfigNodeCount, executorCount;
   private volatile int numExecutorsPerNode = -1;
+  private volatile long memoryPerInstance = -1;
   private LlapRegistryService svc;
   private final Configuration conf;
   private final long updateIntervalNs;
@@ -93,6 +94,14 @@ public class LlapClusterStateForCompile {
     return numExecutorsPerNode;
   }
 
+  public long getMemoryPerInstance() {
+    return memoryPerInstance;
+  }
+
+  public long getMemoryPerExecutor() {
+    return getMemoryPerInstance() / getNumExecutorsPerNode();
+  }
+
   private boolean isUpdateNeeded() {
     Long lastUpdateLocal = lastClusterUpdateNs;
     if (lastUpdateLocal == null) return true;
@@ -134,6 +143,9 @@ public class LlapClusterStateForCompile {
           if (numExecutorsPerNode == -1) {
             numExecutorsPerNode = numExecutors;
           }
+          if (memoryPerInstance == -1) {
+            memoryPerInstance = si.getResource().getMemorySize() * 1024L * 1024L;
+          }
         } catch (NumberFormatException e) {
           ++noConfigNodesLocal;
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index fdc9635..91d2f1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -77,11 +77,14 @@ import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantProp
 import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin;
 import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization;
 import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc;
+import org.apache.hadoop.hive.ql.optimizer.NonBlockingOpDeDupProc;
 import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
 import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize;
 import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
 import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer;
+import org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer;
 import org.apache.hadoop.hive.ql.optimizer.TopNKeyProcessor;
+import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication;
 import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkJoinDeDuplication;
 import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits;
 import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
@@ -170,6 +173,27 @@ public class TezCompiler extends TaskCompiler {
     // Update bucketing version of ReduceSinkOp if needed
     updateBucketingVersionForUpgrade(procCtx);
 
+    // run Sorted dynamic partition optimization
+    if(HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.DYNAMICPARTITIONING) &&
+        HiveConf.getVar(procCtx.conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") &&
+        !HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) {
+      perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
+      new SortedDynPartitionOptimizer().transform(procCtx.parseContext);
+      perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Sorted dynamic partition optimization");
+    }
+
+    if(HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION)
+        || procCtx.parseContext.hasAcidWrite()) {
+      perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
+      // Dynamic sort partition adds an extra RS therefore need to de-dup
+      new ReduceSinkDeDuplication().transform(procCtx.parseContext);
+      // there is an issue with dedup logic wherein SELECT is created with wrong columns
+      // NonBlockingOpDeDupProc fixes that
+      // (kind of hackish, the issue in de-dup should be fixed but it needs more investigation)
+      new NonBlockingOpDeDupProc().transform(procCtx.parseContext);
+      perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Reduce Sink de-duplication");
+    }
+
     perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
     // run the optimizations that use stats for optimization
     runStatsDependentOptimizations(procCtx, inputs, outputs);

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dp_counter_mm.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dp_counter_mm.q b/ql/src/test/queries/clientpositive/dp_counter_mm.q
index 91c4f42..1c655fd 100644
--- a/ql/src/test/queries/clientpositive/dp_counter_mm.q
+++ b/ql/src/test/queries/clientpositive/dp_counter_mm.q
@@ -4,6 +4,7 @@ set hive.exec.max.dynamic.partitions.pernode=200;
 set hive.exec.max.dynamic.partitions=200;
 set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 
 drop table src2_n5;
 create table src2_n5 (key int) partitioned by (value string) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dp_counter_non_mm.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dp_counter_non_mm.q b/ql/src/test/queries/clientpositive/dp_counter_non_mm.q
index 561ae6e..d2da0b0 100644
--- a/ql/src/test/queries/clientpositive/dp_counter_non_mm.q
+++ b/ql/src/test/queries/clientpositive/dp_counter_non_mm.q
@@ -2,6 +2,7 @@
 set hive.exec.dynamic.partition.mode=nonstrict;
 set hive.exec.max.dynamic.partitions.pernode=200;
 set hive.exec.max.dynamic.partitions=200;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 
 drop table src2_n3;
 create table src2_n3 (key int) partitioned by (value string);

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
index 3c2918f..8a3522d 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
@@ -8,8 +8,7 @@ set hive.exec.max.dynamic.partitions=1000;
 set hive.exec.max.dynamic.partitions.pernode=1000;
 set hive.exec.dynamic.partition.mode=nonstrict;
 set hive.vectorized.execution.enabled=true;
-
-
+set hive.optimize.sort.dynamic.partition.threshold=1;
 
 create table over1k_n1(
            t tinyint,
@@ -106,19 +105,19 @@ create table over1k_part2_orc(
            f float)
        partitioned by (ds string, t tinyint);
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i;
 explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_orc order by i limit 10) tmp where t is null or t=27;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 group by si,i,b,f,t;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 -- tests for HIVE-8162, only partition column 't' should be in last RS operator
 explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 group by si,i,b,f,t;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i;
 
 desc formatted over1k_part2_orc partition(ds="foo",t=27);
@@ -128,7 +127,7 @@ desc formatted over1k_part2_orc partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__
 select * from over1k_part2_orc;
 select count(*) from over1k_part2_orc;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i;
 
 desc formatted over1k_part2_orc partition(ds="foo",t=27);
@@ -150,12 +149,12 @@ create table over1k_part_buck_sort2_orc(
        clustered by (si)
        sorted by (f) into 1 buckets;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27;
 
 desc formatted over1k_part_buck_sort2_orc partition(t=27);
@@ -166,7 +165,7 @@ select * from over1k_part_buck_sort2_orc;
 explain select count(*) from over1k_part_buck_sort2_orc;
 select count(*) from over1k_part_buck_sort2_orc;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27;
 
 desc formatted over1k_part_buck_sort2_orc partition(t=27);
@@ -194,7 +193,7 @@ partitioned by (s string)
 clustered by (si) into 2 buckets
 stored as orc tblproperties ('transactional'='true');
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert into table addcolumns_vectorization_true_disallowincompatible_true_fileformat_orc_tinyint partition (s)
   select cint,csmallint, cstring1 from alltypesorc limit 10;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
index 436c0ed..dbeb874 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
@@ -7,8 +7,7 @@ set hive.exec.dynamic.partition=true;
 set hive.exec.max.dynamic.partitions=1000;
 set hive.exec.max.dynamic.partitions.pernode=1000;
 set hive.exec.dynamic.partition.mode=nonstrict;
-
-
+set hive.optimize.sort.dynamic.partition.threshold=1;
 
 create table over1k_n3(
            t tinyint,
@@ -100,19 +99,19 @@ create table over1k_part2(
            f float)
        partitioned by (ds string, t tinyint);
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i;
 explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 -- tests for HIVE-8162, only partition column 't' should be in last RS operator
 explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i;
 
 desc formatted over1k_part2 partition(ds="foo",t=27);
@@ -122,7 +121,7 @@ desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
 select * from over1k_part2;
 select count(*) from over1k_part2;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i;
 
 desc formatted over1k_part2 partition(ds="foo",t=27);
@@ -144,12 +143,12 @@ create table over1k_part_buck_sort2(
        clustered by (si)
        sorted by (f) into 1 buckets;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27;
 
 desc formatted over1k_part_buck_sort2 partition(t=27);
@@ -158,7 +157,7 @@ desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__");
 select * from over1k_part_buck_sort2;
 select count(*) from over1k_part_buck_sort2;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27;
 
 desc formatted over1k_part_buck_sort2 partition(t=27);
@@ -173,7 +172,7 @@ create table over1k_part3(
            f float)
        partitioned by (s string, t tinyint, i int);
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo";
 explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27;
 explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100;
@@ -209,3 +208,41 @@ insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from ov
 insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo";
 
 select sum(hash(*)) from over1k_part3;
+
+drop table over1k_n3;
+create table over1k_n3(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp,
+           `dec` decimal(4,2),
+           bin binary)
+       row format delimited
+       fields terminated by '|';
+
+load data local inpath '../../data/files/over1k' into table over1k_n3;
+
+analyze table over1k_n3 compute statistics for columns;
+set hive.stats.fetch.column.stats=true;
+
+-- default hive should do cost based and add extra RS
+set hive.optimize.sort.dynamic.partition.threshold=0;
+explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27;
+
+-- default but shouldn't add extra RS
+explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10;
+
+-- disable
+set hive.optimize.sort.dynamic.partition.threshold=-1;
+explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27;
+
+-- enable, will add extra RS
+set hive.optimize.sort.dynamic.partition.threshold=1;
+explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10;
+
+drop table over1k_n3;

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q
index 6dfb51a..2f3afb6 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q
@@ -6,8 +6,7 @@ set hive.exec.dynamic.partition=true;
 set hive.exec.max.dynamic.partitions=1000;
 set hive.exec.max.dynamic.partitions.pernode=1000;
 set hive.exec.dynamic.partition.mode=nonstrict;
-
-
+set hive.optimize.sort.dynamic.partition.threshold=1;
 
 -- SORT_QUERY_RESULTS
 
@@ -78,7 +77,7 @@ select * from ss_part where ss_sold_date_sk=2452617;
 desc formatted ss_part partition(ss_sold_date_sk=2452638);
 select * from ss_part where ss_sold_date_sk=2452638;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 -- SORT DYNAMIC PARTITION DISABLED
 
 explain insert overwrite table ss_part partition (ss_sold_date_sk)
@@ -210,7 +209,7 @@ create table if not exists hive13_dp1 (
 PARTITIONED BY(`day` string)
 STORED AS ORC;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 explain insert overwrite table `hive13_dp1` partition(`day`)
 select
     key k1,
@@ -228,7 +227,7 @@ from src
 group by "day", key;
 select * from hive13_dp1 order by k1, k2 limit 5;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 explain insert overwrite table `hive13_dp1` partition(`day`)
 select
     key k1,

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
index d0b32cd..7e27b88 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q
@@ -7,7 +7,7 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 
 set hive.exec.dynamic.partition.mode=nonstrict;
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 
 -- single level partition, sorted dynamic partition disabled
 drop table if exists acid_part;
@@ -31,7 +31,7 @@ select count(*) from acid_part where ds in ('2008-04-08');
 delete from acid_part where key = 'foo' and ds='2008-04-08';
 select count(*) from acid_part where ds='2008-04-08';
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 
 -- single level partition, sorted dynamic partition enabled
 drop table if exists acid_part_sdpo;
@@ -54,7 +54,7 @@ select count(*) from acid_part_sdpo where ds in ('2008-04-08');
 delete from acid_part_sdpo where key = 'foo' and ds='2008-04-08';
 select count(*) from acid_part_sdpo where ds='2008-04-08';
 
-set hive.optimize.sort.dynamic.partition=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 
 -- 2 level partition, sorted dynamic partition disabled
 drop table if exists acid_2L_part;
@@ -83,7 +83,7 @@ delete from acid_2L_part where value = 'bar';
 delete from acid_2L_part where value = 'bar';
 select count(*) from acid_2L_part;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 
 -- 2 level partition, sorted dynamic partition enabled
 drop table if exists acid_2L_part_sdpo;
@@ -113,7 +113,7 @@ delete from acid_2L_part_sdpo where value = 'bar';
 select count(*) from acid_2L_part_sdpo;
 
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 set hive.optimize.constant.propagation=false;
 
 -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled
@@ -137,4 +137,4 @@ select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr>=11;
 delete from acid_2L_part_sdpo_no_cp where key = 'foo' and ds='2008-04-08' and hr=11;
 select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr=11;
 
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q
index c852d51..ca8a442 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q
@@ -2,7 +2,7 @@
 set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 set hive.exec.dynamic.partition.mode=nonstrict;
-set hive.optimize.sort.dynamic.partition=true;
+set hive.optimize.sort.dynamic.partition.threshold=1;
 
 CREATE TABLE non_acid(key string, value string)
 PARTITIONED BY(ds string, hr int)

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/load_data_using_job.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/load_data_using_job.q b/ql/src/test/queries/clientpositive/load_data_using_job.q
index 970a752..a2d45e5 100644
--- a/ql/src/test/queries/clientpositive/load_data_using_job.q
+++ b/ql/src/test/queries/clientpositive/load_data_using_job.q
@@ -9,6 +9,8 @@ set hive.auto.convert.join.noconditionaltask=true;
 set hive.auto.convert.join.noconditionaltask.size=10000;
 set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.TableSizeBasedBigTableSelectorForAutoSMJ;
 
+-- SORT_QUERY_RESULTS
+
 -- Single partition
 -- Regular load happens.
 CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string) STORED AS TEXTFILE;

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/tez_input_counters.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_input_counters.q b/ql/src/test/queries/clientpositive/tez_input_counters.q
index 4461966..1d9d7de 100644
--- a/ql/src/test/queries/clientpositive/tez_input_counters.q
+++ b/ql/src/test/queries/clientpositive/tez_input_counters.q
@@ -8,6 +8,7 @@ set hive.fetch.task.conversion=none;
 set hive.map.aggr=false;
 -- disabling map side aggregation as that can lead to different intermediate record counts
 set hive.tez.exec.print.summary=true;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
 
 create table testpart (k int) partitioned by (v string);
 insert overwrite table testpart partition(v) select * from src;

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q b/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q
index 74ca011..eaa714e 100644
--- a/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q
+++ b/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q
@@ -4,7 +4,6 @@ set hive.fetch.task.conversion=none;
 
 -- Check if vectorization code is handling partitioning on DATE and the other data types.
 
-
 CREATE TABLE flights_tiny_n1 (
   origin_city_name STRING,
   dest_city_name STRING,
@@ -19,6 +18,8 @@ CREATE TABLE flights_tiny_orc STORED AS ORC AS
 SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num
 FROM flights_tiny_n1;
 
+-- SORT_QUERY_RESULTS
+
 SELECT * FROM flights_tiny_orc;
 
 SET hive.vectorized.execution.enabled=false;


[2/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out
index 7cd5ba4..a912216 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out
@@ -76,52 +76,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1_n0
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1_n0
 
   Stage: Stage-2
     Dependency Collection
@@ -164,7 +140,7 @@ POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).key EXPRESSION [(src
 POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-Found 6 items
+Found 1 items
 #### A masked pattern was here ####
 PREHOOK: query: EXPLAIN
     INSERT OVERWRITE TABLE orcfile_merge1b_n0 PARTITION (ds='1', part)
@@ -207,52 +183,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1b_n0
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1b_n0
 
   Stage: Stage-8
     Conditional Operator
@@ -383,52 +335,28 @@ STAGE PLANS:
                     expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int)
                     outputColumnNames: _col0, _col1, _col2
                     Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orcfile_merge1c_n0
-                    Select Operator
-                      expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string)
-                      outputColumnNames: key, value, ds, part
-                      Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE
-                      Group By Operator
-                        aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
-                        keys: ds (type: string), part (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string)
-                          sort order: ++
-                          Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
-                          Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col2 (type: int)
+                      sort order: +
+                      Map-reduce partition columns: _col2 (type: int)
+                      value expressions: _col0 (type: int), _col1 (type: string)
             Execution mode: llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
-                keys: KEY._col0 (type: string), KEY._col1 (type: string)
-                mode: mergepartial
-                outputColumnNames: _col0, _col1, _col2, _col3
-                Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                Select Operator
-                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: string)
-                  outputColumnNames: _col0, _col1, _col2, _col3
-                  Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              Select Operator
+                expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int)
+                outputColumnNames: _col0, _col1, _col2
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orcfile_merge1c_n0
 
   Stage: Stage-8
     Conditional Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
index 5c5bc8e..ee13faf 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
@@ -52,52 +52,28 @@ STAGE PLANS:
                     expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double)
                     outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
                     Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.orc_merge5a
-                    Select Operator
-                      expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double)
-                      outputColumnNames: userid, string1, subtype, decimal1, ts, st
-                      Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                      Group By Operator
-                        aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll')
-                        keys: st (type: double)
-                        mode: hash
-                        outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                        Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: double)
-                          sort order: +
-                          Map-reduce partition columns: _col0 (type: double)
-                          Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                          value expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:binary>), _col4 (type: struct<columntype:string,min:decimal(38,0),max:decimal(38,0),countnulls:bigint,bitvector:binary>), _col5 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>)
+                    Reduce Output Operator
+                      key expressions: _col5 (type: double)
+                      sort order: +
+                      Map-reduce partition columns: _col5 (type: double)
+                      value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp)
             Execution mode: llap
             LLAP IO: all inputs
         Reducer 2 
             Execution mode: llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4)
-                keys: KEY._col0 (type: double)
-                mode: mergepartial
+              Select Operator
+                expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col4 (type: struct<columntype:string,min:decimal(38,0),max:decimal(38,0),countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col5 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: double)
-                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
                   Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  table:
+                      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+                      serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+                      name: default.orc_merge5a
 
   Stage: Stage-2
     Dependency Collection

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
index 9d7b988..d981733 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
@@ -2594,24 +2594,16 @@ STAGE PLANS:
                       sort order: 
                       Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
                       value expressions: _col0 (type: bigint), _col1 (type: bigint)
-                Select Operator
-                  expressions: _col1 (type: double)
-                  outputColumnNames: _col1
-                  Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
-                  Select Operator
-                    expressions: floor(_col1) (type: bigint)
+                  Group By Operator
+                    keys: _col0 (type: bigint)
+                    mode: hash
                     outputColumnNames: _col0
-                    Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
-                    Group By Operator
-                      keys: _col0 (type: bigint)
-                      mode: hash
-                      outputColumnNames: _col0
+                    Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+                    Reduce Output Operator
+                      key expressions: _col0 (type: bigint)
+                      sort order: +
+                      Map-reduce partition columns: _col0 (type: bigint)
                       Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
-                      Reduce Output Operator
-                        key expressions: _col0 (type: bigint)
-                        sort order: +
-                        Map-reduce partition columns: _col0 (type: bigint)
-                        Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
         Reducer 6 
             Execution mode: vectorized, llap
             Reduce Operator Tree:

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/tez_dml.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/tez_dml.q.out b/ql/src/test/results/clientpositive/llap/tez_dml.q.out
index 67d4bf3..a9b3172 100644
--- a/ql/src/test/results/clientpositive/llap/tez_dml.q.out
+++ b/ql/src/test/results/clientpositive/llap/tez_dml.q.out
@@ -466,52 +466,28 @@ STAGE PLANS:
                     expressions: value (type: string), cnt (type: bigint)
                     outputColumnNames: _col0, _col1
                     Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.mapred.TextInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          name: default.tmp_src_part
-                    Select Operator
-                      expressions: _col0 (type: string), UDFToInteger(_col1) (type: int)
-                      outputColumnNames: c, d
-                      Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE
-                      Group By Operator
-                        aggregations: compute_stats(c, 'hll')
-                        keys: d (type: int)
-                        mode: hash
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: int)
-                          sort order: +
-                          Map-reduce partition columns: _col0 (type: int)
-                          Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE
-                          value expressions: _col1 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
-            Execution mode: llap
+                    Reduce Output Operator
+                      key expressions: _col1 (type: bigint)
+                      sort order: +
+                      Map-reduce partition columns: _col1 (type: bigint)
+                      value expressions: _col0 (type: string)
+            Execution mode: vectorized, llap
             LLAP IO: no inputs
         Reducer 2 
-            Execution mode: llap
+            Execution mode: vectorized, llap
             Reduce Operator Tree:
-              Group By Operator
-                aggregations: compute_stats(VALUE._col0)
-                keys: KEY._col0 (type: int)
-                mode: mergepartial
+              Select Operator
+                expressions: VALUE._col0 (type: string), KEY._col1 (type: bigint)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 154 Data size: 28228 Basic stats: COMPLETE Column stats: NONE
-                Select Operator
-                  expressions: _col1 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: int)
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 154 Data size: 28228 Basic stats: COMPLETE Column stats: NONE
-                  File Output Operator
-                    compressed: false
-                    Statistics: Num rows: 154 Data size: 28228 Basic stats: COMPLETE Column stats: NONE
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.tmp_src_part
 
   Stage: Stage-2
     Dependency Collection

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out b/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out
index ca8232e..28977d7 100644
--- a/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out
@@ -1316,7 +1316,7 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: web_sales
-                  Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE
                   TableScan Vectorization:
                       native: true
                   Select Operator
@@ -1326,7 +1326,7 @@ STAGE PLANS:
                         className: VectorSelectOperator
                         native: true
                         projectedOutputColumnNums: [16]
-                    Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE
                     Group By Operator
                       Group By Vectorization:
                           className: VectorGroupByOperator
@@ -1338,7 +1338,7 @@ STAGE PLANS:
                       keys: ws_order_number (type: int)
                       mode: hash
                       outputColumnNames: _col0
-                      Statistics: Num rows: 170 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE
                       Reduce Output Operator
                         key expressions: _col0 (type: int)
                         sort order: +
@@ -1347,7 +1347,7 @@ STAGE PLANS:
                             className: VectorReduceSinkLongOperator
                             native: true
                             nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
-                        Statistics: Num rows: 170 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE
             Execution mode: vectorized, llap
             LLAP IO: all inputs
             Map Vectorization:
@@ -1379,7 +1379,7 @@ STAGE PLANS:
                 keys: KEY._col0 (type: int)
                 mode: mergepartial
                 outputColumnNames: _col0
-                Statistics: Num rows: 170 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 1000 Data size: 1755802 Basic stats: COMPLETE Column stats: NONE
                 Group By Operator
                   aggregations: count(_col0)
                   Group By Vectorization:
@@ -1391,14 +1391,14 @@ STAGE PLANS:
                       projectedOutputColumnNums: [0]
                   mode: hash
                   outputColumnNames: _col0
-                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
                   Reduce Output Operator
                     sort order: 
                     Reduce Sink Vectorization:
                         className: VectorReduceSinkEmptyKeyOperator
                         native: true
                         nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
-                    Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
                     value expressions: _col0 (type: bigint)
         Reducer 3 
             Execution mode: vectorized, llap
@@ -1420,13 +1420,13 @@ STAGE PLANS:
                     projectedOutputColumnNums: [0]
                 mode: mergepartial
                 outputColumnNames: _col0
-                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   File Sink Vectorization:
                       className: VectorFileSinkOperator
                       native: false
-                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat


[6/7] hive git commit: HIVE-20703: Put dynamic sort partition optimization under cost based decision (Vineet Garg, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/acid_table_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out
index e1af88a..e06624c 100644
--- a/ql/src/test/results/clientpositive/acid_table_stats.q.out
+++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out
@@ -97,7 +97,7 @@ Partition Parameters:
 	numFiles            	2                   
 	numRows             	1000                
 	rawDataSize         	0                   
-	totalSize           	4063                
+	totalSize           	4542                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -184,7 +184,7 @@ Partition Parameters:
 	numFiles            	2                   
 	numRows             	1000                
 	rawDataSize         	0                   
-	totalSize           	4063                
+	totalSize           	4542                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -235,7 +235,7 @@ Partition Parameters:
 	numFiles            	2                   
 	numRows             	1000                
 	rawDataSize         	0                   
-	totalSize           	4063                
+	totalSize           	4542                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -331,7 +331,7 @@ Partition Parameters:
 	numFiles            	4                   
 	numRows             	2000                
 	rawDataSize         	0                   
-	totalSize           	8126                
+	totalSize           	9085                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -380,7 +380,7 @@ Partition Parameters:
 	numFiles            	4                   
 	numRows             	2000                
 	rawDataSize         	0                   
-	totalSize           	8126                
+	totalSize           	9085                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -455,11 +455,11 @@ STAGE PLANS:
           TableScan
             alias: acid
             filterExpr: (ds = '2008-04-08') (type: boolean)
-            Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               expressions: key (type: string)
               outputColumnNames: key
-              Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 aggregations: max(key)
                 mode: hash
@@ -591,7 +591,7 @@ Partition Parameters:
 	numFiles            	2                   
 	numRows             	1000                
 	rawDataSize         	176000              
-	totalSize           	3008                
+	totalSize           	3485                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
index 20f88bd..5bc9442 100644
--- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
+++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out
@@ -31,8 +31,10 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
-  Stage-2 depends on stages: Stage-0
+  Stage-2 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0, Stage-4
+  Stage-4 depends on stages: Stage-2
 
 STAGE PLANS:
   Stage: Stage-1
@@ -46,26 +48,62 @@ STAGE PLANS:
               outputColumnNames: _col0, _col1, _col2, _col3
               Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
               Reduce Output Operator
-                key expressions: _col2 (type: string), _col3 (type: string), _bucket_number (type: string), _col1 (type: string)
-                sort order: ++++
-                Map-reduce partition columns: _col2 (type: string), _col3 (type: string)
+                key expressions: _col1 (type: string)
+                sort order: +
                 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
-                value expressions: _col0 (type: string)
+                value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string)
       Execution mode: vectorized
       Reduce Operator Tree:
         Select Operator
-          expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string)
-          outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number
+          expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
+          outputColumnNames: _col0, _col1, _col2, _col3
+          Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-2
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Reduce Output Operator
+              sort order: 
+              Map-reduce partition columns: _col0 (type: string)
+              Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+              value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+      Execution mode: vectorized
+      Reduce Operator Tree:
+        Select Operator
+          expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string)
+          outputColumnNames: _col0, _col1, _col2, _col3
           Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
-            Dp Sort State: PARTITION_BUCKET_SORTED
             Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
             table:
                 input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                 output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
                 serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
                 name: default.non_acid
+          Select Operator
+            expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), UDFToInteger(_col3) (type: int)
+            outputColumnNames: key, value, ds, hr
+            Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+            Group By Operator
+              aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
+              keys: ds (type: string), hr (type: int)
+              mode: hash
+              outputColumnNames: _col0, _col1, _col2, _col3
+              Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+              File Output Operator
+                compressed: false
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-0
     Move Operator
@@ -80,7 +118,7 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.non_acid
 
-  Stage: Stage-2
+  Stage: Stage-3
     Stats Work
       Basic Stats Work:
       Column Stats Desc:
@@ -88,3 +126,33 @@ STAGE PLANS:
           Column Types: string, string
           Table: default.non_acid
 
+  Stage: Stage-4
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Reduce Output Operator
+              key expressions: _col0 (type: string), _col1 (type: int)
+              sort order: ++
+              Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+              Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+              value expressions: _col2 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,sumlength:bigint,count:bigint,countnulls:bigint,bitvector:binary>)
+      Execution mode: vectorized
+      Reduce Operator Tree:
+        Group By Operator
+          aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1)
+          keys: KEY._col0 (type: string), KEY._col1 (type: int)
+          mode: mergepartial
+          outputColumnNames: _col0, _col1, _col2, _col3
+          Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: _col2 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,maxlength:bigint,avglength:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: int)
+            outputColumnNames: _col0, _col1, _col2, _col3
+            Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out
index 0414812..2c6d7ca 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out
@@ -223,16 +223,16 @@ STAGE PLANS:
                 TableScan
                   alias: bucket_small_n17
                   filterExpr: pri BETWEEN 1 AND 2 (type: boolean)
-                  Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
                   Select Operator
                     expressions: key (type: bigint), value (type: string)
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
                     Reduce Output Operator
                       key expressions: _col0 (type: bigint)
                       sort order: +
                       Map-reduce partition columns: _col0 (type: bigint)
-                      Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
                       value expressions: _col1 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
@@ -246,14 +246,14 @@ STAGE PLANS:
                   0 _col0 (type: bigint)
                   1 _col0 (type: bigint)
                 outputColumnNames: _col0, _col1, _col3
-                Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
                   expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int)
                   outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                  Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                   File Output Operator
                     compressed: false
-                    Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                     table:
                         input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                         output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -371,11 +371,11 @@ STAGE PLANS:
                 TableScan
                   alias: bucket_small_n17
                   filterExpr: pri BETWEEN 1 AND 2 (type: boolean)
-                  Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
                   Select Operator
                     expressions: key (type: bigint), value (type: string)
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
             Map Operator Tree:
                 TableScan
                   alias: bucket_big_n17
@@ -392,14 +392,14 @@ STAGE PLANS:
                         0 _col0 (type: bigint)
                         1 _col0 (type: bigint)
                       outputColumnNames: _col0, _col1, _col3
-                      Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int)
                         outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                        Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                         File Output Operator
                           compressed: false
-                          Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                           table:
                               input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                               output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -627,11 +627,11 @@ STAGE PLANS:
                 TableScan
                   alias: bucket_small_n17
                   filterExpr: pri BETWEEN 1 AND 2 (type: boolean)
-                  Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
                   Select Operator
                     expressions: key (type: bigint), value (type: string)
                     outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE
             Map Operator Tree:
                 TableScan
                   alias: bucket_big_n17
@@ -648,14 +648,14 @@ STAGE PLANS:
                         0 _col0 (type: bigint)
                         1 _col0 (type: bigint)
                       outputColumnNames: _col0, _col1, _col3
-                      Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int)
                         outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                        Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                         File Output Operator
                           compressed: false
-                          Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE
+                          Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE
                           table:
                               input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                               output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
index 62fdbdb..bd3c776 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out
@@ -189,7 +189,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _col0 (type: smallint)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -199,7 +198,6 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -275,7 +273,6 @@ STAGE PLANS:
                         Reduce Output Operator
                           sort order: 
                           Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
-                          TopN Hash Memory Usage: 0.1
                           value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -293,7 +290,6 @@ STAGE PLANS:
                     key expressions: _col4 (type: tinyint)
                     sort order: +
                     Map-reduce partition columns: _col4 (type: tinyint)
-                    Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
         Reducer 3 
             Execution mode: vectorized, llap
@@ -301,7 +297,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -374,7 +369,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -384,11 +378,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -456,7 +449,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float)
                         sort order: +++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -466,11 +458,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -606,7 +597,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _col0 (type: smallint)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -616,7 +606,6 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -692,7 +681,6 @@ STAGE PLANS:
                         Reduce Output Operator
                           sort order: 
                           Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
-                          TopN Hash Memory Usage: 0.1
                           value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -710,7 +698,6 @@ STAGE PLANS:
                     key expressions: _col4 (type: tinyint)
                     sort order: +
                     Map-reduce partition columns: _col4 (type: tinyint)
-                    Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
         Reducer 3 
             Execution mode: vectorized, llap
@@ -718,7 +705,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -791,7 +777,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -801,11 +786,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -873,7 +857,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float)
                         sort order: +++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -883,11 +866,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -1171,7 +1153,7 @@ Partition Parameters:
 	numFiles            	8                   
 	numRows             	32                  
 	rawDataSize         	640                 
-	totalSize           	4648                
+	totalSize           	4670                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -1249,7 +1231,7 @@ Partition Parameters:
 	numFiles            	8                   
 	numRows             	32                  
 	rawDataSize         	640                 
-	totalSize           	4658                
+	totalSize           	4656                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -1527,7 +1509,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _col1 (type: int)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -1537,7 +1518,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -1607,7 +1587,6 @@ STAGE PLANS:
                       key expressions: _col2 (type: int)
                       sort order: +
                       Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE
-                      TopN Hash Memory Usage: 0.1
                       value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -1632,7 +1611,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint)
                         sort order: +
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
         Reducer 3 
             Execution mode: vectorized, llap
@@ -1640,7 +1618,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -2347,7 +2324,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float)
                         sort order: +++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -2357,11 +2333,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -2906,7 +2881,6 @@ STAGE PLANS:
                       Reduce Output Operator
                         sort order: 
                         Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
-                        TopN Hash Memory Usage: 0.1
                         value expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: string)
             Execution mode: vectorized, llap
             LLAP IO: all inputs
@@ -2924,7 +2898,6 @@ STAGE PLANS:
                     key expressions: _col2 (type: string), _bucket_number (type: string)
                     sort order: ++
                     Map-reduce partition columns: _col2 (type: string)
-                    Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col0 (type: int), _col1 (type: smallint)
         Reducer 3 
             Execution mode: vectorized, llap
@@ -2932,11 +2905,10 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: int), VALUE._col1 (type: smallint), KEY._col2 (type: string), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _bucket_number
-                Statistics: Num rows: 10 Data size: 2656 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
-                  Statistics: Num rows: 10 Data size: 2656 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
index ec95389..bf29a11 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out
@@ -146,7 +146,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint)
                         sort order: +
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -156,7 +155,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -232,7 +230,6 @@ STAGE PLANS:
                         Reduce Output Operator
                           sort order: 
                           Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
-                          TopN Hash Memory Usage: 0.1
                           value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint)
             Execution mode: llap
             LLAP IO: no inputs
@@ -250,7 +247,6 @@ STAGE PLANS:
                     key expressions: _col4 (type: tinyint)
                     sort order: +
                     Map-reduce partition columns: _col4 (type: tinyint)
-                    Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                     value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
         Reducer 3 
             Execution mode: llap
@@ -258,7 +254,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -331,7 +326,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -341,7 +335,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
@@ -413,7 +406,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float)
                         sort order: +++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint)
             Execution mode: llap
             LLAP IO: no inputs
@@ -423,7 +415,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
@@ -563,7 +554,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint)
                         sort order: +
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -573,7 +563,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -649,7 +638,6 @@ STAGE PLANS:
                         Reduce Output Operator
                           sort order: 
                           Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
-                          TopN Hash Memory Usage: 0.1
                           value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint)
             Execution mode: llap
             LLAP IO: no inputs
@@ -667,7 +655,6 @@ STAGE PLANS:
                     key expressions: _col4 (type: tinyint)
                     sort order: +
                     Map-reduce partition columns: _col4 (type: tinyint)
-                    Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                     value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
         Reducer 3 
             Execution mode: llap
@@ -675,7 +662,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -748,7 +734,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -758,7 +743,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
@@ -830,7 +814,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float)
                         sort order: +++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint)
             Execution mode: llap
             LLAP IO: no inputs
@@ -840,7 +823,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
@@ -1484,7 +1466,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _col1 (type: int)
                         sort order: ++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -1494,7 +1475,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -1564,7 +1544,6 @@ STAGE PLANS:
                       key expressions: _col2 (type: int)
                       sort order: +
                       Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
-                      TopN Hash Memory Usage: 0.1
                       value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -1589,7 +1568,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint)
                         sort order: +
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
         Reducer 3 
             Execution mode: llap
@@ -1597,7 +1575,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -2304,7 +2281,6 @@ STAGE PLANS:
                         key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float)
                         sort order: +++
                         Map-reduce partition columns: _col4 (type: tinyint)
-                        Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint)
             Execution mode: llap
             LLAP IO: no inputs
@@ -2314,7 +2290,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_BUCKET_SORTED
@@ -2680,7 +2655,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
-                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -2690,7 +2664,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -2764,7 +2737,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
-                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -2774,7 +2746,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -2848,7 +2819,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
-                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -2858,7 +2828,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -2932,7 +2901,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
-                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -2942,7 +2910,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -3016,7 +2983,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
-                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -3026,7 +2992,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -3100,7 +3065,6 @@ STAGE PLANS:
                         key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
                         sort order: +++
                         Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int)
-                        Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float)
             Execution mode: llap
             LLAP IO: no inputs
@@ -3110,7 +3074,6 @@ STAGE PLANS:
               Select Operator
                 expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-                Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED
@@ -3584,3 +3547,455 @@ POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
 POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
 #### A masked pattern was here ####
 17814641134
+PREHOOK: query: drop table over1k_n3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_n3
+POSTHOOK: query: drop table over1k_n3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@over1k_n3
+POSTHOOK: Output: default@over1k_n3
+PREHOOK: query: create table over1k_n3(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp,
+           `dec` decimal(4,2),
+           bin binary)
+       row format delimited
+       fields terminated by '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over1k_n3
+POSTHOOK: query: create table over1k_n3(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp,
+           `dec` decimal(4,2),
+           bin binary)
+       row format delimited
+       fields terminated by '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over1k_n3
+PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@over1k_n3
+POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@over1k_n3
+PREHOOK: query: analyze table over1k_n3 compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_n3
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table over1k_n3 compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@over1k_n3
+POSTHOOK: Output: default@over1k_n3
+#### A masked pattern was here ####
+PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_part@ds=foo
+POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_n3
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: over1k_n3
+                  filterExpr: (t is null or (t > 27Y)) (type: boolean)
+                  Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: ((t > 27Y) or t is null) (type: boolean)
+                    Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+                      outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                      Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        key expressions: _col4 (type: tinyint)
+                        sort order: +
+                        Map-reduce partition columns: _col4 (type: tinyint)
+                        value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.over1k_part
+
+  Stage: Stage-2
+    Dependency Collection
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          partition:
+            ds foo
+            t 
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.over1k_part
+
+  Stage: Stage-3
+    Stats Work
+      Basic Stats Work:
+      Column Stats Desc:
+          Columns: si, i, b, f
+          Column Types: smallint, int, bigint, float
+          Table: default.over1k_part
+
+PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_part@ds=foo
+POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_n3
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+        Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: over1k_n3
+                  filterExpr: (t is null or (t = 27Y)) (type: boolean)
+                  Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: ((t = 27Y) or t is null) (type: boolean)
+                    Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+                      outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                      Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
+                      Limit
+                        Number of rows: 10
+                        Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                        Reduce Output Operator
+                          sort order: 
+                          Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                          value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                Limit
+                  Number of rows: 10
+                  Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                  Reduce Output Operator
+                    key expressions: _col4 (type: tinyint)
+                    sort order: +
+                    Map-reduce partition columns: _col4 (type: tinyint)
+                    value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
+        Reducer 3 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.over1k_part
+
+  Stage: Stage-2
+    Dependency Collection
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          partition:
+            ds foo
+            t 
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.over1k_part
+
+  Stage: Stage-3
+    Stats Work
+      Basic Stats Work:
+      Column Stats Desc:
+          Columns: si, i, b, f
+          Column Types: smallint, int, bigint, float
+          Table: default.over1k_part
+
+PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_part@ds=foo
+POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_n3
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: over1k_n3
+                  filterExpr: (t is null or (t > 27Y)) (type: boolean)
+                  Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: ((t > 27Y) or t is null) (type: boolean)
+                    Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+                      outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                      Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE
+                      File Output Operator
+                        compressed: false
+                        Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE
+                        table:
+                            input format: org.apache.hadoop.mapred.TextInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                            name: default.over1k_part
+                      Select Operator
+                        expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint)
+                        outputColumnNames: si, i, b, f, ds, t
+                        Statistics: Num rows: 352 Data size: 39072 Basic stats: COMPLETE Column stats: COMPLETE
+                        Group By Operator
+                          aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll')
+                          keys: ds (type: string), t (type: tinyint)
+                          mode: hash
+                          outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                          Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: string), _col1 (type: tinyint)
+                            sort order: ++
+                            Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint)
+                            Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE
+                            value expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col3 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col4 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,bitvector:binary>), _col5 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,bitvector:binary>)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3)
+                keys: KEY._col0 (type: string), KEY._col1 (type: tinyint)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE
+                Select Operator
+                  expressions: _col2 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col3 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col4 (type: struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col5 (type: struct<columntype:string,min:double,max:double,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>), _col0 (type: string), _col1 (type: tinyint)
+                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                  Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-2
+    Dependency Collection
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          partition:
+            ds foo
+            t 
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.over1k_part
+
+  Stage: Stage-3
+    Stats Work
+      Basic Stats Work:
+      Column Stats Desc:
+          Columns: si, i, b, f
+          Column Types: smallint, int, bigint, float
+          Table: default.over1k_part
+
+PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_part@ds=foo
+POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_n3
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-2 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+        Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: over1k_n3
+                  filterExpr: (t is null or (t = 27Y)) (type: boolean)
+                  Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: ((t = 27Y) or t is null) (type: boolean)
+                    Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint)
+                      outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                      Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE
+                      Limit
+                        Number of rows: 10
+                        Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                        Reduce Output Operator
+                          sort order: 
+                          Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                          value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint)
+            Execution mode: llap
+            LLAP IO: no inputs
+        Reducer 2 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                Limit
+                  Number of rows: 10
+                  Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                  Reduce Output Operator
+                    key expressions: _col4 (type: tinyint)
+                    sort order: +
+                    Map-reduce partition columns: _col4 (type: tinyint)
+                    value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float)
+        Reducer 3 
+            Execution mode: llap
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                File Output Operator
+                  compressed: false
+                  Dp Sort State: PARTITION_SORTED
+                  Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.over1k_part
+
+  Stage: Stage-2
+    Dependency Collection
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          partition:
+            ds foo
+            t 
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.over1k_part
+
+  Stage: Stage-3
+    Stats Work
+      Basic Stats Work:
+      Column Stats Desc:
+          Columns: si, i, b, f
+          Column Types: smallint, int, bigint, float
+          Table: default.over1k_part
+
+PREHOOK: query: drop table over1k_n3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@over1k_n3
+PREHOOK: Output: default@over1k_n3
+POSTHOOK: query: drop table over1k_n3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@over1k_n3
+POSTHOOK: Output: default@over1k_n3

http://git-wip-us.apache.org/repos/asf/hive/blob/5eebbdf7/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
index d953eeb..30074ab 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
@@ -352,19 +352,17 @@ STAGE PLANS:
                       Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
                       Reduce Output Operator
                         key expressions: _col2 (type: int)
-                        sort order: +
+                        sort order: ++
                         Map-reduce partition columns: _col2 (type: int)
-                        Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
-                        value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int)
+                        value expressions: _col0 (type: float), _col1 (type: float)
             Execution mode: vectorized, llap
             LLAP IO: no inputs
         Reducer 2 
             Execution mode: vectorized, llap
             Reduce Operator Tree:
               Select Operator
-                expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), VALUE._col2 (type: int)
+                expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), KEY._col2 (type: int)
                 outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
                   Dp Sort State: PARTITION_SORTED