You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/28 05:59:52 UTC
svn commit: r1642263 [3/3] - in /hive/branches/spark: ./ data/conf/
data/conf/spark/ itests/qtest-spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/
ql/src/test/queries/clientpositive/ ql/s...
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out Fri Nov 28 04:59:50 2014
@@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@smb_bucket_3
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -377,10 +381,10 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key right outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -488,12 +492,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -508,7 +512,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -587,16 +591,16 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
-NULL NULL NULL NULL 4 val_4
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
NULL NULL NULL NULL 20 val_20
NULL NULL NULL NULL 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -924,12 +928,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
+NULL NULL 20 val_20 20 val_20
+NULL NULL 23 val_23 23 val_23
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
-NULL NULL 20 val_20 20 val_20
-NULL NULL 23 val_23 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -944,7 +948,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1022,14 +1026,14 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 19 val_19
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
NULL NULL 30 val_30 NULL NULL
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -1044,7 +1048,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1138,7 +1142,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1217,10 +1221,10 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
@@ -1239,7 +1243,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1317,12 +1321,12 @@ POSTHOOK: Input: default@smb_bucket_1
POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
-NULL NULL NULL NULL 4 val_4
+NULL NULL 20 val_20 20 val_20
+NULL NULL 23 val_23 23 val_23
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 17 val_17
NULL NULL NULL NULL 19 val_19
-NULL NULL 20 val_20 20 val_20
-NULL NULL 23 val_23 23 val_23
+NULL NULL NULL NULL 4 val_4
PREHOOK: query: explain
select /*+mapjoin(a,c)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key
PREHOOK: type: QUERY
@@ -1337,7 +1341,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1416,15 +1420,15 @@ POSTHOOK: Input: default@smb_bucket_2
POSTHOOK: Input: default@smb_bucket_3
#### A masked pattern was here ####
1 val_1 NULL NULL NULL NULL
+10 val_10 NULL NULL NULL NULL
3 val_3 NULL NULL NULL NULL
4 val_4 NULL NULL NULL NULL
-NULL NULL NULL NULL 4 val_4
5 val_5 NULL NULL NULL NULL
-10 val_10 NULL NULL NULL NULL
-NULL NULL NULL NULL 10 val_10
-NULL NULL NULL NULL 17 val_17
-NULL NULL NULL NULL 19 val_19
NULL NULL 20 val_20 20 val_20
NULL NULL 23 val_23 23 val_23
NULL NULL 25 val_25 NULL NULL
NULL NULL 30 val_30 NULL NULL
+NULL NULL NULL NULL 10 val_10
+NULL NULL NULL NULL 17 val_17
+NULL NULL NULL NULL 19 val_19
+NULL NULL NULL NULL 4 val_4
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out Fri Nov 28 04:59:50 2014
@@ -623,7 +623,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out Fri Nov 28 04:59:50 2014
@@ -1,8 +1,12 @@
-PREHOOK: query: create table smb_bucket_input (key int, value string) stored as rcfile
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table smb_bucket_input (key int, value string) stored as rcfile
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@smb_bucket_input
-POSTHOOK: query: create table smb_bucket_input (key int, value string) stored as rcfile
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table smb_bucket_input (key int, value string) stored as rcfile
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@smb_bucket_input
@@ -68,11 +72,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
-4 val_356 NULL NULL
-NULL NULL 484 val_169
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
+4 val_356 NULL NULL
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
+NULL NULL 484 val_169
NULL NULL 5000 val_125
PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
PREHOOK: type: QUERY
@@ -84,11 +88,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
-4 val_356 NULL NULL
-NULL NULL 484 val_169
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
+4 val_356 NULL NULL
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
+NULL NULL 484 val_169
NULL NULL 5000 val_125
PREHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=2000 or key=4000
PREHOOK: type: QUERY
@@ -121,8 +125,8 @@ POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
NULL NULL 5000 val_125
PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
PREHOOK: type: QUERY
@@ -135,8 +139,8 @@ POSTHOOK: Input: default@smb_bucket4_1
POSTHOOK: Input: default@smb_bucket4_2
#### A masked pattern was here ####
2000 val_169 NULL NULL
-NULL NULL 3000 val_169
4000 val_125 NULL NULL
+NULL NULL 3000 val_169
NULL NULL 5000 val_125
PREHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=4000
PREHOOK: type: QUERY
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_in.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_in.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_in.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_in.q.out Fri Nov 28 04:59:50 2014
@@ -281,7 +281,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3)
Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -441,8 +441,8 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
+ Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3)
+ Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -609,7 +609,7 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -829,7 +829,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union25.q.out Fri Nov 28 04:59:50 2014
@@ -68,8 +68,8 @@ STAGE PLANS:
Edges:
Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0)
Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Union 2 (GROUP, 1)
- Reducer 5 <- Union 4 (GROUP, 1)
+ Reducer 3 <- Union 2 (GROUP, 3)
+ Reducer 5 <- Union 4 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union33.q.out Fri Nov 28 04:59:50 2014
@@ -196,9 +196,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out Fri Nov 28 04:59:50 2014
@@ -276,8 +276,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
@@ -448,8 +448,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 5 <- Map 4 (GROUP, 3)
Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/uniquejoin.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/uniquejoin.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/uniquejoin.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/uniquejoin.q.out Fri Nov 28 04:59:50 2014
@@ -46,14 +46,18 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@t3
-PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
SELECT a.key, b.key, c.key
PREHOOK: type: QUERY
PREHOOK: Input: default@t1
PREHOOK: Input: default@t2
PREHOOK: Input: default@t3
#### A masked pattern was here ####
-POSTHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key)
SELECT a.key, b.key, c.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t1
@@ -63,14 +67,14 @@ POSTHOOK: Input: default@t3
1 NULL NULL
2 2 2
3 3 NULL
-NULL 4 4
-NULL 5 NULL
-NULL NULL 6
7 NULL 7
8 8 NULL
8 8 NULL
8 8 NULL
8 8 NULL
+NULL 4 4
+NULL 5 NULL
+NULL NULL 6
PREHOOK: query: FROM UNIQUEJOIN T1 a (a.key), T2 b (b.key), T3 c (c.key)
SELECT a.key, b.key, c.key
PREHOOK: type: QUERY
@@ -119,15 +123,15 @@ POSTHOOK: Input: default@t3
#### A masked pattern was here ####
1 11 NULL NULL NULL NULL
2 12 NULL NULL 2 12
-NULL NULL 2 22 NULL NULL
3 13 3 13 NULL NULL
-NULL NULL 4 14 4 14
-NULL NULL 5 15 NULL NULL
-NULL NULL NULL NULL 6 16
7 17 NULL NULL 7 17
8 18 8 18 NULL NULL
8 18 8 18 NULL NULL
8 28 NULL NULL NULL NULL
+NULL NULL 2 22 NULL NULL
+NULL NULL 4 14 4 14
+NULL NULL 5 15 NULL NULL
+NULL NULL NULL NULL 6 16
PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b (b.key), PRESERVE T3 c (c.key)
SELECT a.key, b.key, c.key
PREHOOK: type: QUERY
@@ -145,13 +149,13 @@ POSTHOOK: Input: default@t3
1 NULL NULL
2 2 2
3 3 NULL
-NULL 4 4
-NULL NULL 6
7 NULL 7
8 8 NULL
8 8 NULL
8 8 NULL
8 8 NULL
+NULL 4 4
+NULL NULL 6
PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b(b.key)
SELECT a.key, b.key
PREHOOK: type: QUERY
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out Fri Nov 28 04:59:50 2014
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out Fri Nov 28 04:59:50 2014
@@ -16,7 +16,9 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@alltypesorc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@decimal_vgby
-PREHOOK: query: -- First only do simple aggregations that output primitives only
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+-- First only do simple aggregations that output primitives only
EXPLAIN SELECT cint,
COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1),
COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2)
@@ -24,7 +26,9 @@ EXPLAIN SELECT cint,
GROUP BY cint
HAVING COUNT(*) > 1
PREHOOK: type: QUERY
-POSTHOOK: query: -- First only do simple aggregations that output primitives only
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+-- First only do simple aggregations that output primitives only
EXPLAIN SELECT cint,
COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1),
COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2)
@@ -40,7 +44,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -113,13 +117,13 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby
#### A masked pattern was here ####
-762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925
-253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089
-3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 6 6984454.211097692 -4033.445769230769 6967702.8672438458471
-563 2 -515.621072973 -3367.6517567568 -3883.2728297298 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459
-6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462
-626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746
+253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089
528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003
+626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746
+6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462
+762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925
NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536
PREHOOK: query: -- Now add the others...
EXPLAIN SELECT cint,
@@ -145,7 +149,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -217,11 +221,11 @@ POSTHOOK: query: SELECT cint,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@decimal_vgby
#### A masked pattern was here ####
-762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758
-253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274
-3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 1161283.811207307641183333 2604201.2704476737 2852759.5602156054
-563 2 -515.621072973 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -2325.50327307692295 1707.9424961538462 2415.395441814127
-6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653
-626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603
+253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274
528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984
+626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603
+6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653
+762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758
NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out Fri Nov 28 04:59:50 2014
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out Fri Nov 28 04:59:50 2014
@@ -119,7 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out Fri Nov 28 04:59:50 2014
@@ -257,7 +257,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 1)
+ Reducer 3 <- Map 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
Map 2
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out Fri Nov 28 04:59:50 2014
@@ -115,7 +115,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_13.q.out Fri Nov 28 04:59:50 2014
@@ -28,6 +28,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
@@ -60,6 +61,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -71,6 +73,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -107,19 +110,31 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double)
+ outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
+ sort order: +++++
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -156,6 +171,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
@@ -189,50 +205,51 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != 12)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL -31 1969-12-31 16:00:16.059 -31.0 NULL 31 -31 0 -31.0 -0.0 31.0 -2466.143 0.0 31.0 0.0 -10.175 -31.0 0.847741935483871 -31.0 0.0 -31
-true 50 1969-12-31 16:00:16.338 50.0 821UdmGbkEf4j -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50
-NULL 23 1969-12-31 16:00:11.785 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23
-true 14 1969-12-31 16:00:11.528 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14
-true 50 1969-12-31 16:00:11.087 50.0 1cGVWH7n1QU -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50
-NULL 13 1969-12-31 16:00:12.498 13.0 NULL -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13
-true 36 1969-12-31 16:00:15.145 36.0 1cGVWH7n1QU -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36
-true 3 1969-12-31 16:00:14.349 3.0 cvLH6Eat2yFsyy7p -3 3 0 3.0 0.0 -3.0 238.659 0.0 -3.0 0.0 -10.175 3.0 -8.76 3.0 0.0 3
-NULL -4 1969-12-31 16:00:13.237 -4.0 NULL 4 -4 0 -4.0 -0.0 4.0 -318.212 0.0 4.0 0.0 -10.175 -4.0 6.57 -4.0 0.0 -4
-NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20
-true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30
-true -57 1969-12-31 16:00:13.365 -57.0 1cGVWH7n1QU 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
-true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40
-NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21
-NULL 7 1969-12-31 16:00:14.872 7.0 NULL -7 7 0 7.0 0.0 -7.0 556.871 0.0 -7.0 0.0 -10.175 7.0 -3.7542857142857144 7.0 0.0 7
-true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34
-NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
-true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
-NULL 44 1969-12-31 16:00:16.279 44.0 NULL -44 44 0 44.0 0.0 -44.0 3500.332 0.0 -44.0 0.0 -10.175 44.0 -0.5972727272727273 44.0 0.0 44
-true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
-NULL 0 1969-12-31 16:00:13.396 0.0 NULL 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0
-true -25 1969-12-31 16:00:15.516 -25.0 821UdmGbkEf4j 25 -25 0 -25.0 -0.0 25.0 -1988.8249999999998 0.0 25.0 0.0 -10.175 -25.0 1.0512000000000001 -25.0 0.0 -25
-true -40 1969-12-31 16:00:11.833 -40.0 cvLH6Eat2yFsyy7p 40 -40 0 -40.0 -0.0 40.0 -3182.12 0.0 40.0 0.0 -10.175 -40.0 0.657 -40.0 0.0 -40
-NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29
-NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
+NULL -64 1969-12-31 16:00:11.912 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:12.339 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:13.274 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -63 1969-12-31 16:00:11.946 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:12.188 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
-NULL -6 1969-12-31 16:00:11.883 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6
-true 14 1969-12-31 16:00:11.302 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14
-true -63 1969-12-31 16:00:14.899 -63.0 cvLH6Eat2yFsyy7p 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
-true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13
-true 49 1969-12-31 16:00:15.541 49.0 cvLH6Eat2yFsyy7p -49 49 0 49.0 0.0 -49.0 3898.0969999999998 0.0 -49.0 0.0 -10.175 49.0 -0.5363265306122449 49.0 0.0 49
-true -60 1969-12-31 16:00:12.291 -60.0 821UdmGbkEf4j 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
-NULL 32 1969-12-31 16:00:14.026 32.0 NULL -32 32 0 32.0 0.0 -32.0 2545.696 0.0 -32.0 0.0 -10.175 32.0 -0.82125 32.0 0.0 32
-true 16 1969-12-31 16:00:15.698 16.0 1cGVWH7n1QU -16 16 0 16.0 0.0 -16.0 1272.848 0.0 -16.0 0.0 -10.175 16.0 -1.6425 16.0 0.0 16
-NULL 57 1969-12-31 16:00:14.011 57.0 NULL -57 57 0 57.0 0.0 -57.0 4534.521 0.0 -57.0 0.0 -10.175 57.0 -0.4610526315789474 57.0 0.0 57
-true 41 1969-12-31 16:00:12.235 41.0 cvLH6Eat2yFsyy7p -41 41 0 41.0 0.0 -41.0 3261.673 0.0 -41.0 0.0 -10.175 41.0 -0.6409756097560976 41.0 0.0 41
-true -64 1969-12-31 16:00:12.857 -64.0 821UdmGbkEf4j 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
-true 36 1969-12-31 16:00:13.825 36.0 cvLH6Eat2yFsyy7p -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36
-NULL -15 1969-12-31 16:00:15.186 -15.0 NULL 15 -15 0 -15.0 -0.0 15.0 -1193.295 0.0 15.0 0.0 -10.175 -15.0 1.752 -15.0 0.0 -15
-true -2 1969-12-31 16:00:15.431 -2.0 1cGVWH7n1QU 2 -2 0 -2.0 -0.0 2.0 -159.106 0.0 2.0 0.0 -10.175 -2.0 13.14 -2.0 0.0 -2
+NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:12.591 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.154 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.517 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.965 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -61 1969-12-31 16:00:11.842 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:12.454 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:14.192 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:16.558 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -60 1969-12-31 16:00:11.504 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -60 1969-12-31 16:00:11.641 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -60 1969-12-31 16:00:11.996 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -60 1969-12-31 16:00:12.779 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60
+NULL -59 1969-12-31 16:00:12.008 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:13.15 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:13.625 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:15.296 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -59 1969-12-31 16:00:15.861 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
+NULL -58 1969-12-31 16:00:12.065 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:12.683 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:12.948 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:14.066 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
+NULL -57 1969-12-31 16:00:11.451 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:11.883 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:12.626 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:13.578 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -57 1969-12-31 16:00:15.39 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
+NULL -56 1969-12-31 16:00:13.602 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56
+NULL -56 1969-12-31 16:00:13.958 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56
+NULL -56 1969-12-31 16:00:15.038 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56
+NULL -55 1969-12-31 16:00:11.38 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
+NULL -55 1969-12-31 16:00:11.751 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
PREHOOK: query: -- double compare timestamp
EXPLAIN
SELECT cboolean1,
@@ -264,6 +281,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
POSTHOOK: query: -- double compare timestamp
@@ -297,6 +315,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -307,7 +326,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
+ Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -344,19 +364,31 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double)
+ outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
+ Reduce Output Operator
+ key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
+ sort order: +++++
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
Stage: Stage-0
Fetch Operator
@@ -393,6 +425,7 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
@@ -426,47 +459,48 @@ WHERE (((cfloat < 3569)
AND ((ctimestamp2 != -1.3359999999999999)
AND (ctinyint < 9763215.5639))))
GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
+ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1
LIMIT 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL -26 1969-12-31 16:00:08.33 -26.0 NULL 26 -26 0 -26.0 -0.0 26.0 -2068.3779999999997 0.0 26.0 0.0 -10.175 -26.0 1.0107692307692309 -26.0 0.0 -26
-NULL -6 1969-12-31 16:00:07.423 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6
-true 55 1969-12-31 15:59:58.969 55.0 cvLH6Eat2yFsyy7p -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55
-true 14 1969-12-31 16:00:08.209 14.0 1cGVWH7n1QU -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14
-true 48 1969-12-31 16:00:00.923 48.0 1cGVWH7n1QU -48 48 0 48.0 0.0 -48.0 3818.544 0.0 -48.0 0.0 -10.175 48.0 -0.5475 48.0 0.0 48
-true -10 1969-12-31 16:00:00.082 -10.0 1cGVWH7n1QU 10 -10 0 -10.0 -0.0 10.0 -795.53 0.0 10.0 0.0 -10.175 -10.0 2.628 -10.0 0.0 -10
-true -55 1969-12-31 16:00:05.591 -55.0 1cGVWH7n1QU 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
-NULL -35 1969-12-31 16:00:04.52 -35.0 NULL 35 -35 0 -35.0 -0.0 35.0 -2784.355 0.0 35.0 0.0 -10.175 -35.0 0.7508571428571429 -35.0 0.0 -35
-NULL -8 1969-12-31 16:00:00.65 -8.0 NULL 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8
-NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20
-true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30
-true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40
-NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21
-true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34
-NULL 23 1969-12-31 16:00:08.307 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23
-true 8 1969-12-31 16:00:10.496 8.0 821UdmGbkEf4j -8 8 0 8.0 0.0 -8.0 636.424 0.0 -8.0 0.0 -10.175 8.0 -3.285 8.0 0.0 8
-NULL -55 1969-12-31 16:00:08.896 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55
-NULL -57 1969-12-31 16:00:02.707 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57
-true 1 1969-12-31 16:00:00.125 1.0 cvLH6Eat2yFsyy7p -1 1 0 1.0 0.0 -1.0 79.553 0.0 -1.0 0.0 -10.175 1.0 -26.28 1.0 0.0 1
-true -1 1969-12-31 16:00:00.107 -1.0 1cGVWH7n1QU 1 -1 0 -1.0 -0.0 1.0 -79.553 0.0 1.0 0.0 -10.175 -1.0 26.28 -1.0 0.0 -1
-true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59
-NULL 33 1969-12-31 16:00:07.006 33.0 NULL -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33
-NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58
-true -6 1969-12-31 16:00:07.372 -6.0 cvLH6Eat2yFsyy7p 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6
-true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
-NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29
-NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27
-true 33 1969-12-31 16:00:09.924 33.0 cvLH6Eat2yFsyy7p -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33
-NULL -18 1969-12-31 16:00:00.564 -18.0 NULL 18 -18 0 -18.0 -0.0 18.0 -1431.954 0.0 18.0 0.0 -10.175 -18.0 1.46 -18.0 0.0 -18
-NULL -51 1969-12-31 16:00:02.011 -51.0 NULL 51 -51 0 -51.0 -0.0 51.0 -4057.203 0.0 51.0 0.0 -10.175 -51.0 0.5152941176470588 -51.0 0.0 -51
-true -8 1969-12-31 16:00:02.109 -8.0 cvLH6Eat2yFsyy7p 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8
-true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13
-true -52 1969-12-31 16:00:01.824 -52.0 821UdmGbkEf4j 52 -52 0 -52.0 -0.0 52.0 -4136.755999999999 0.0 52.0 0.0 -10.175 -52.0 0.5053846153846154 -52.0 0.0 -52
-true -28 1969-12-31 16:00:10.554 -28.0 821UdmGbkEf4j 28 -28 0 -28.0 -0.0 28.0 -2227.484 0.0 28.0 0.0 -10.175 -28.0 0.9385714285714286 -28.0 0.0 -28
-true 2 1969-12-31 16:00:06.546 2.0 821UdmGbkEf4j -2 2 0 2.0 0.0 -2.0 159.106 0.0 -2.0 0.0 -10.175 2.0 -13.14 2.0 0.0 2
-true 40 1969-12-31 16:00:02.842 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40
-NULL 55 1969-12-31 16:00:02.215 55.0 NULL -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55
-NULL 22 1969-12-31 16:00:00.621 22.0 NULL -22 22 0 22.0 0.0 -22.0 1750.166 0.0 -22.0 0.0 -10.175 22.0 -1.1945454545454546 22.0 0.0 22
-NULL 58 1969-12-31 16:00:03.366 58.0 NULL -58 58 0 58.0 0.0 -58.0 4614.074 0.0 -58.0 0.0 -10.175 58.0 -0.4531034482758621 58.0 0.0 58
-true 0 1969-12-31 16:00:08.742 0.0 821UdmGbkEf4j 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0
+NULL -64 1969-12-31 16:00:00.199 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:00.29 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:01.785 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:03.944 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:05.997 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:10.858 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:11.912 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:12.339 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -64 1969-12-31 16:00:13.274 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64
+NULL -63 1969-12-31 16:00:01.843 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:03.552 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:06.852 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:07.375 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:10.205 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:11.946 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:12.188 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63
+NULL -62 1969-12-31 16:00:00.037 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:01.22 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:01.515 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:01.734 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:02.373 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:03.85 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:08.198 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:09.025 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:09.889 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:10.069 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:10.225 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:10.485 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:12.591 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.154 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.517 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -62 1969-12-31 16:00:14.965 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62
+NULL -61 1969-12-31 16:00:00.142 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:02.698 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:03.049 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:04.165 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
+NULL -61 1969-12-31 16:00:04.977 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out Fri Nov 28 04:59:50 2014
@@ -1866,7 +1866,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2074,7 +2074,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2329,7 +2329,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
@@ -2662,7 +2662,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 3)
Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out Fri Nov 28 04:59:50 2014 differ
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java Fri Nov 28 04:59:50 2014
@@ -64,7 +64,7 @@ final class ClientUtils {
int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600);
double akkaFailureDetector =
toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0);
- int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000);
+ int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100);
// Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22
// due to packaging changes (relocated org.objenesis classes).
Modified: hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1642263&r1=1642262&r2=1642263&view=diff
==============================================================================
--- hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java (original)
+++ hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java Fri Nov 28 04:59:50 2014
@@ -17,10 +17,12 @@
package org.apache.hive.spark.client;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Serializable;
@@ -192,8 +194,12 @@ class SparkClientImpl implements SparkCl
// If a Spark installation is provided, use the spark-submit script. Otherwise, call the
// SparkSubmit class directly, which has some caveats (like having to provide a proper
// version of Guava on the classpath depending on the deploy mode).
- if (conf.get("spark.home") != null) {
- argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath());
+ String sparkHome = conf.get("spark.home");
+ if (sparkHome == null) {
+ sparkHome = System.getProperty("spark.home");
+ }
+ if (sparkHome != null) {
+ argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath());
} else {
LOG.info("No spark.home provided, calling SparkSubmit directly.");
argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath());
@@ -248,12 +254,11 @@ class SparkClientImpl implements SparkCl
LOG.debug("Running client driver with argv: {}", Joiner.on(" ").join(argv));
ProcessBuilder pb = new ProcessBuilder(argv.toArray(new String[argv.size()]));
- pb.environment().clear();
final Process child = pb.start();
int childId = childIdGenerator.incrementAndGet();
- redirect("stdout-redir-" + childId, child.getInputStream(), System.out);
- redirect("stderr-redir-" + childId, child.getErrorStream(), System.err);
+ redirect("stdout-redir-" + childId, child.getInputStream());
+ redirect("stderr-redir-" + childId, child.getErrorStream());
runnable = new Runnable() {
@Override
@@ -277,8 +282,8 @@ class SparkClientImpl implements SparkCl
return thread;
}
- private void redirect(String name, InputStream in, OutputStream out) {
- Thread thread = new Thread(new Redirector(in, out));
+ private void redirect(String name, InputStream in) {
+ Thread thread = new Thread(new Redirector(in));
thread.setName(name);
thread.setDaemon(true);
thread.start();
@@ -346,23 +351,18 @@ class SparkClientImpl implements SparkCl
private class Redirector implements Runnable {
- private final InputStream in;
- private final OutputStream out;
+ private final BufferedReader in;
- Redirector(InputStream in, OutputStream out) {
- this.in = in;
- this.out = out;
+ Redirector(InputStream in) {
+ this.in = new BufferedReader(new InputStreamReader(in));
}
@Override
public void run() {
try {
- byte[] buf = new byte[1024];
- int len = in.read(buf);
- while (len != -1) {
- out.write(buf, 0, len);
- out.flush();
- len = in.read(buf);
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ LOG.info(line);
}
} catch (Exception e) {
LOG.warn("Error in redirector thread.", e);