You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/12/15 18:11:42 UTC

svn commit: r1642997 [24/42] - in /hive/branches/spark: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/ ql/src/test/results/clientpositive/spark/

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/optimize_nullscan.q.out Tue Dec  2 19:57:10 2014
@@ -214,18 +214,88 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
       Edges:
-        Reducer 3 <- Map 2 (GROUP, 1)
+        Reducer 4 <- Map 3 (GROUP, 1)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Reducer 4 (PARTITION-LEVEL SORT, 3)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+                  GatherStats: false
+                  Filter Operator
+                    isSamplingPred: false
+                    predicate: false (type: boolean)
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: string)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                        tag: 0
+                        auto parallelism: false
+            Path -> Alias:
+              -mr-10003default.src{} [src]
+            Path -> Partition:
+              -mr-10003default.src{} 
+                Partition
+                  base file name: src
+                  input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  properties:
+                    COLUMN_STATS_ACCURATE true
+                    bucket_count -1
+                    columns key,value
+                    columns.comments defaultdefault
+                    columns.types string:string
+#### A masked pattern was here ####
+                    name default.src
+                    numFiles 1
+                    numRows 500
+                    rawDataSize 5312
+                    serialization.ddl struct src { string key, string value}
+                    serialization.format 1
+                    serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
+                    totalSize 5812
+#### A masked pattern was here ####
+                  serde: org.apache.hadoop.hive.serde2.NullStructSerDe
+                
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    properties:
+                      COLUMN_STATS_ACCURATE true
+                      bucket_count -1
+                      columns key,value
+                      columns.comments defaultdefault
+                      columns.types string:string
+#### A masked pattern was here ####
+                      name default.src
+                      numFiles 1
+                      numRows 500
+                      rawDataSize 5312
+                      serialization.ddl struct src { string key, string value}
+                      serialization.format 1
+                      serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      totalSize 5812
+#### A masked pattern was here ####
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: default.src
+                  name: default.src
+            Truncated Path -> Alias:
+              -mr-10003default.src{} [src]
+        Map 3 
             Map Operator Tree:
                 TableScan
                   alias: srcpart
@@ -245,12 +315,12 @@ STAGE PLANS:
                         value expressions: _col0 (type: string)
                         auto parallelism: false
             Path -> Alias:
-              -mr-10005default.srcpart{ds=2008-04-08, hr=11} [srcpart]
-              -mr-10006default.srcpart{ds=2008-04-08, hr=12} [srcpart]
-              -mr-10007default.srcpart{ds=2008-04-09, hr=11} [srcpart]
-              -mr-10008default.srcpart{ds=2008-04-09, hr=12} [srcpart]
+              -mr-10004default.srcpart{ds=2008-04-08, hr=11} [srcpart]
+              -mr-10005default.srcpart{ds=2008-04-08, hr=12} [srcpart]
+              -mr-10006default.srcpart{ds=2008-04-09, hr=11} [srcpart]
+              -mr-10007default.srcpart{ds=2008-04-09, hr=12} [srcpart]
             Path -> Partition:
-              -mr-10005default.srcpart{ds=2008-04-08, hr=11} 
+              -mr-10004default.srcpart{ds=2008-04-08, hr=11} 
                 Partition
                   base file name: hr=11
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -296,7 +366,7 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: default.srcpart
                   name: default.srcpart
-              -mr-10006default.srcpart{ds=2008-04-08, hr=12} 
+              -mr-10005default.srcpart{ds=2008-04-08, hr=12} 
                 Partition
                   base file name: hr=12
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -342,7 +412,7 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: default.srcpart
                   name: default.srcpart
-              -mr-10007default.srcpart{ds=2008-04-09, hr=11} 
+              -mr-10006default.srcpart{ds=2008-04-09, hr=11} 
                 Partition
                   base file name: hr=11
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -388,7 +458,7 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: default.srcpart
                   name: default.srcpart
-              -mr-10008default.srcpart{ds=2008-04-09, hr=12} 
+              -mr-10007default.srcpart{ds=2008-04-09, hr=12} 
                 Partition
                   base file name: hr=12
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -435,13 +505,47 @@ STAGE PLANS:
                     name: default.srcpart
                   name: default.srcpart
             Truncated Path -> Alias:
-              -mr-10005default.srcpart{ds=2008-04-08, hr=11} [srcpart]
-              -mr-10006default.srcpart{ds=2008-04-08, hr=12} [srcpart]
-              -mr-10007default.srcpart{ds=2008-04-09, hr=11} [srcpart]
-              -mr-10008default.srcpart{ds=2008-04-09, hr=12} [srcpart]
-        Reducer 3 
-            Local Work:
-              Map Reduce Local Work
+              -mr-10004default.srcpart{ds=2008-04-08, hr=11} [srcpart]
+              -mr-10005default.srcpart{ds=2008-04-08, hr=12} [srcpart]
+              -mr-10006default.srcpart{ds=2008-04-09, hr=11} [srcpart]
+              -mr-10007default.srcpart{ds=2008-04-09, hr=12} [srcpart]
+        Reducer 2 
+            Needs Tagging: true
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Left Outer Join0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0}
+                  1 {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: string), _col1 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+#### A masked pattern was here ####
+                    NumFilesPerFileSink: 1
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+#### A masked pattern was here ####
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        properties:
+                          columns _col0,_col1
+                          columns.types string:string
+                          escape.delim \
+                          hive.serialization.extend.nesting.levels true
+                          serialization.format 1
+                          serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    TotalFiles: 1
+                    GatherStats: false
+                    MultiFileSpray: false
+        Reducer 4 
             Needs Tagging: false
             Reduce Operator Tree:
               Select Operator
@@ -451,123 +555,13 @@ STAGE PLANS:
                 Limit
                   Number of rows: 0
                   Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Spark HashTable Sink Operator
-                    condition expressions:
-                      0 {_col0}
-                      1 
-                    keys:
-                      0 _col0 (type: string)
-                      1 _col0 (type: string)
-                    Position of Big Table: 0
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: src
-                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
-                  GatherStats: false
-                  Filter Operator
-                    isSamplingPred: false
-                    predicate: false (type: boolean)
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string)
+                    sort order: +
+                    Map-reduce partition columns: _col0 (type: string)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Select Operator
-                      expressions: key (type: string)
-                      outputColumnNames: _col0
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Left Outer Join0 to 1
-                        condition expressions:
-                          0 {_col0}
-                          1 {_col0}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                        outputColumnNames: _col0, _col1
-                        input vertices:
-                          1 Reducer 3
-                        Position of Big Table: 0
-                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                        Select Operator
-                          expressions: _col0 (type: string), _col1 (type: string)
-                          outputColumnNames: _col0, _col1
-                          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                          File Output Operator
-                            compressed: false
-                            GlobalTableId: 0
-#### A masked pattern was here ####
-                            NumFilesPerFileSink: 1
-                            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-#### A masked pattern was here ####
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                properties:
-                                  columns _col0,_col1
-                                  columns.types string:string
-                                  escape.delim \
-                                  hive.serialization.extend.nesting.levels true
-                                  serialization.format 1
-                                  serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                            TotalFiles: 1
-                            GatherStats: false
-                            MultiFileSpray: false
-            Local Work:
-              Map Reduce Local Work
-            Path -> Alias:
-              -mr-10004default.src{} [src]
-            Path -> Partition:
-              -mr-10004default.src{} 
-                Partition
-                  base file name: src
-                  input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
-                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                  properties:
-                    COLUMN_STATS_ACCURATE true
-                    bucket_count -1
-                    columns key,value
-                    columns.comments defaultdefault
-                    columns.types string:string
-#### A masked pattern was here ####
-                    name default.src
-                    numFiles 1
-                    numRows 500
-                    rawDataSize 5312
-                    serialization.ddl struct src { string key, string value}
-                    serialization.format 1
-                    serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
-                    totalSize 5812
-#### A masked pattern was here ####
-                  serde: org.apache.hadoop.hive.serde2.NullStructSerDe
-                
-                    input format: org.apache.hadoop.mapred.TextInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                    properties:
-                      COLUMN_STATS_ACCURATE true
-                      bucket_count -1
-                      columns key,value
-                      columns.comments defaultdefault
-                      columns.types string:string
-#### A masked pattern was here ####
-                      name default.src
-                      numFiles 1
-                      numRows 500
-                      rawDataSize 5312
-                      serialization.ddl struct src { string key, string value}
-                      serialization.format 1
-                      serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                      totalSize 5812
-#### A masked pattern was here ####
-                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                    name: default.src
-                  name: default.src
-            Truncated Path -> Alias:
-              -mr-10004default.src{} [src]
+                    tag: 1
+                    auto parallelism: false
 
   Stage: Stage-0
     Fetch Operator
@@ -1087,18 +1081,87 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
       Edges:
-        Reducer 3 <- Map 2 (GROUP, 1)
+        Reducer 4 <- Map 3 (GROUP, 1)
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+                  GatherStats: false
+                  Filter Operator
+                    isSamplingPred: false
+                    predicate: false (type: boolean)
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: string)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                        tag: 0
+                        value expressions: _col0 (type: string)
+                        auto parallelism: false
+            Path -> Alias:
+              -mr-10003default.src{} [src]
+            Path -> Partition:
+              -mr-10003default.src{} 
+                Partition
+                  base file name: src
+                  input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  properties:
+                    COLUMN_STATS_ACCURATE true
+                    bucket_count -1
+                    columns key,value
+                    columns.comments defaultdefault
+                    columns.types string:string
+#### A masked pattern was here ####
+                    name default.src
+                    numFiles 1
+                    numRows 500
+                    rawDataSize 5312
+                    serialization.ddl struct src { string key, string value}
+                    serialization.format 1
+                    serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
+                    totalSize 5812
+#### A masked pattern was here ####
+                  serde: org.apache.hadoop.hive.serde2.NullStructSerDe
+                
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    properties:
+                      COLUMN_STATS_ACCURATE true
+                      bucket_count -1
+                      columns key,value
+                      columns.comments defaultdefault
+                      columns.types string:string
+#### A masked pattern was here ####
+                      name default.src
+                      numFiles 1
+                      numRows 500
+                      rawDataSize 5312
+                      serialization.ddl struct src { string key, string value}
+                      serialization.format 1
+                      serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      totalSize 5812
+#### A masked pattern was here ####
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: default.src
+                  name: default.src
+            Truncated Path -> Alias:
+              -mr-10003default.src{} [src]
+        Map 3 
             Map Operator Tree:
                 TableScan
                   alias: srcpart
@@ -1118,12 +1181,12 @@ STAGE PLANS:
                         value expressions: _col0 (type: string)
                         auto parallelism: false
             Path -> Alias:
-              -mr-10005default.srcpart{ds=2008-04-08, hr=11} [srcpart]
-              -mr-10006default.srcpart{ds=2008-04-08, hr=12} [srcpart]
-              -mr-10007default.srcpart{ds=2008-04-09, hr=11} [srcpart]
-              -mr-10008default.srcpart{ds=2008-04-09, hr=12} [srcpart]
+              -mr-10004default.srcpart{ds=2008-04-08, hr=11} [srcpart]
+              -mr-10005default.srcpart{ds=2008-04-08, hr=12} [srcpart]
+              -mr-10006default.srcpart{ds=2008-04-09, hr=11} [srcpart]
+              -mr-10007default.srcpart{ds=2008-04-09, hr=12} [srcpart]
             Path -> Partition:
-              -mr-10005default.srcpart{ds=2008-04-08, hr=11} 
+              -mr-10004default.srcpart{ds=2008-04-08, hr=11} 
                 Partition
                   base file name: hr=11
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -1169,7 +1232,7 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: default.srcpart
                   name: default.srcpart
-              -mr-10006default.srcpart{ds=2008-04-08, hr=12} 
+              -mr-10005default.srcpart{ds=2008-04-08, hr=12} 
                 Partition
                   base file name: hr=12
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -1215,7 +1278,7 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: default.srcpart
                   name: default.srcpart
-              -mr-10007default.srcpart{ds=2008-04-09, hr=11} 
+              -mr-10006default.srcpart{ds=2008-04-09, hr=11} 
                 Partition
                   base file name: hr=11
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -1261,7 +1324,7 @@ STAGE PLANS:
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: default.srcpart
                   name: default.srcpart
-              -mr-10008default.srcpart{ds=2008-04-09, hr=12} 
+              -mr-10007default.srcpart{ds=2008-04-09, hr=12} 
                 Partition
                   base file name: hr=12
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -1308,13 +1371,47 @@ STAGE PLANS:
                     name: default.srcpart
                   name: default.srcpart
             Truncated Path -> Alias:
-              -mr-10005default.srcpart{ds=2008-04-08, hr=11} [srcpart]
-              -mr-10006default.srcpart{ds=2008-04-08, hr=12} [srcpart]
-              -mr-10007default.srcpart{ds=2008-04-09, hr=11} [srcpart]
-              -mr-10008default.srcpart{ds=2008-04-09, hr=12} [srcpart]
-        Reducer 3 
-            Local Work:
-              Map Reduce Local Work
+              -mr-10004default.srcpart{ds=2008-04-08, hr=11} [srcpart]
+              -mr-10005default.srcpart{ds=2008-04-08, hr=12} [srcpart]
+              -mr-10006default.srcpart{ds=2008-04-09, hr=11} [srcpart]
+              -mr-10007default.srcpart{ds=2008-04-09, hr=12} [srcpart]
+        Reducer 2 
+            Needs Tagging: true
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Left Outer Join0 to 1
+                condition expressions:
+                  0 {VALUE._col0}
+                  1 {VALUE._col0}
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: string), _col1 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+#### A masked pattern was here ####
+                    NumFilesPerFileSink: 1
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+#### A masked pattern was here ####
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        properties:
+                          columns _col0,_col1
+                          columns.types string:string
+                          escape.delim \
+                          hive.serialization.extend.nesting.levels true
+                          serialization.format 1
+                          serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    TotalFiles: 1
+                    GatherStats: false
+                    MultiFileSpray: false
+        Reducer 4 
             Needs Tagging: false
             Reduce Operator Tree:
               Select Operator
@@ -1324,123 +1421,12 @@ STAGE PLANS:
                 Limit
                   Number of rows: 0
                   Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  Spark HashTable Sink Operator
-                    condition expressions:
-                      0 {_col0}
-                      1 {_col0}
-                    keys:
-                      0 
-                      1 
-                    Position of Big Table: 0
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: src
-                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
-                  GatherStats: false
-                  Filter Operator
-                    isSamplingPred: false
-                    predicate: false (type: boolean)
+                  Reduce Output Operator
+                    sort order: 
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Select Operator
-                      expressions: key (type: string)
-                      outputColumnNames: _col0
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Left Outer Join0 to 1
-                        condition expressions:
-                          0 {_col0}
-                          1 {_col0}
-                        keys:
-                          0 
-                          1 
-                        outputColumnNames: _col0, _col1
-                        input vertices:
-                          1 Reducer 3
-                        Position of Big Table: 0
-                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                        Select Operator
-                          expressions: _col0 (type: string), _col1 (type: string)
-                          outputColumnNames: _col0, _col1
-                          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                          File Output Operator
-                            compressed: false
-                            GlobalTableId: 0
-#### A masked pattern was here ####
-                            NumFilesPerFileSink: 1
-                            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-#### A masked pattern was here ####
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                properties:
-                                  columns _col0,_col1
-                                  columns.types string:string
-                                  escape.delim \
-                                  hive.serialization.extend.nesting.levels true
-                                  serialization.format 1
-                                  serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                            TotalFiles: 1
-                            GatherStats: false
-                            MultiFileSpray: false
-            Local Work:
-              Map Reduce Local Work
-            Path -> Alias:
-              -mr-10004default.src{} [src]
-            Path -> Partition:
-              -mr-10004default.src{} 
-                Partition
-                  base file name: src
-                  input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
-                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                  properties:
-                    COLUMN_STATS_ACCURATE true
-                    bucket_count -1
-                    columns key,value
-                    columns.comments defaultdefault
-                    columns.types string:string
-#### A masked pattern was here ####
-                    name default.src
-                    numFiles 1
-                    numRows 500
-                    rawDataSize 5312
-                    serialization.ddl struct src { string key, string value}
-                    serialization.format 1
-                    serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
-                    totalSize 5812
-#### A masked pattern was here ####
-                  serde: org.apache.hadoop.hive.serde2.NullStructSerDe
-                
-                    input format: org.apache.hadoop.mapred.TextInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                    properties:
-                      COLUMN_STATS_ACCURATE true
-                      bucket_count -1
-                      columns key,value
-                      columns.comments defaultdefault
-                      columns.types string:string
-#### A masked pattern was here ####
-                      name default.src
-                      numFiles 1
-                      numRows 500
-                      rawDataSize 5312
-                      serialization.ddl struct src { string key, string value}
-                      serialization.format 1
-                      serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                      totalSize 5812
-#### A masked pattern was here ####
-                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                    name: default.src
-                  name: default.src
-            Truncated Path -> Alias:
-              -mr-10004default.src{} [src]
+                    tag: 1
+                    value expressions: _col0 (type: string)
+                    auto parallelism: false
 
   Stage: Stage-0
     Fetch Operator
@@ -1532,23 +1518,15 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 4 
-            Local Work:
-              Map Reduce Local Work
-
   Stage: Stage-1
     Spark
       Edges:
-        Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0)
+        Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
+        Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -1638,55 +1616,26 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: src
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: false (type: boolean)
-                    Map Join Operator
-                      condition map:
-                           Left Outer Join0 to 1
-                      condition expressions:
-                        0 {key}
-                        1 
-                      keys:
-                        0 key (type: string)
-                        1 key (type: string)
-                      outputColumnNames: _col0
-                      input vertices:
-                        1 Map 4
-                      Position of Big Table: 0
-                      Select Operator
-                        expressions: _col0 (type: string)
-                        outputColumnNames: _col0
-                        File Output Operator
-                          compressed: false
-                          GlobalTableId: 0
-#### A masked pattern was here ####
-                          NumFilesPerFileSink: 1
-#### A masked pattern was here ####
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              properties:
-                                columns _col0
-                                columns.types string
-                                escape.delim \
-                                hive.serialization.extend.nesting.levels true
-                                serialization.format 1
-                                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          TotalFiles: 1
-                          GatherStats: false
-                          MultiFileSpray: false
-            Local Work:
-              Map Reduce Local Work
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: key (type: string)
+                      sort order: +
+                      Map-reduce partition columns: key (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      tag: 0
+                      auto parallelism: false
             Path -> Alias:
-#### A masked pattern was here ####
+              -mr-10003default.src{} [src]
             Path -> Partition:
-#### A masked pattern was here ####
+              -mr-10003default.src{} 
                 Partition
                   base file name: src
-                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                   properties:
                     COLUMN_STATS_ACCURATE true
@@ -1701,10 +1650,10 @@ STAGE PLANS:
                     rawDataSize 5312
                     serialization.ddl struct src { string key, string value}
                     serialization.format 1
-                    serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
                     totalSize 5812
 #### A masked pattern was here ####
-                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                  serde: org.apache.hadoop.hive.serde2.NullStructSerDe
                 
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -1728,7 +1677,41 @@ STAGE PLANS:
                     name: default.src
                   name: default.src
             Truncated Path -> Alias:
-              /src [src]
+              -mr-10003default.src{} [src]
+        Map 5 
+        Reducer 4 
+            Needs Tagging: true
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Left Outer Join0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0}
+                  1 
+                outputColumnNames: _col0
+                Select Operator
+                  expressions: _col0 (type: string)
+                  outputColumnNames: _col0
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+#### A masked pattern was here ####
+                    NumFilesPerFileSink: 1
+#### A masked pattern was here ####
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        properties:
+                          columns _col0
+                          columns.types string
+                          escape.delim \
+                          hive.serialization.extend.nesting.levels true
+                          serialization.format 1
+                          serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    TotalFiles: 1
+                    GatherStats: false
+                    MultiFileSpray: false
         Union 2 
             Vertex: Union 2
 
@@ -1789,39 +1772,38 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
             Map Operator Tree:
                 TableScan
-                  alias: s2
+                  alias: s1
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: false (type: boolean)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {key} {value}
-                        1 {key}
-                      keys:
-                        0 value (type: string)
-                        1 value (type: string)
-                      Position of Big Table: 0
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      key expressions: value (type: string)
+                      sort order: +
+                      Map-reduce partition columns: value (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      tag: 0
+                      value expressions: key (type: string)
+                      auto parallelism: false
             Path -> Alias:
-              -mr-10005default.src{} [s2]
+              -mr-10004default.src{} [s1]
             Path -> Partition:
-              -mr-10005default.src{} 
+              -mr-10004default.src{} 
                 Partition
                   base file name: src
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -1866,67 +1848,29 @@ STAGE PLANS:
                     name: default.src
                   name: default.src
             Truncated Path -> Alias:
-              -mr-10005default.src{} [s2]
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+              -mr-10004default.src{} [s1]
+        Map 3 
             Map Operator Tree:
                 TableScan
-                  alias: s1
+                  alias: s2
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: false (type: boolean)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Map Join Operator
-                      condition map:
-                           Inner Join 0 to 1
-                      condition expressions:
-                        0 {key} {value}
-                        1 {key} {value}
-                      keys:
-                        0 value (type: string)
-                        1 value (type: string)
-                      outputColumnNames: _col0, _col1, _col5, _col6
-                      input vertices:
-                        1 Map 2
-                      Position of Big Table: 0
+                    Reduce Output Operator
+                      key expressions: value (type: string)
+                      sort order: +
+                      Map-reduce partition columns: value (type: string)
                       Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Select Operator
-                        expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                        File Output Operator
-                          compressed: false
-                          GlobalTableId: 0
-#### A masked pattern was here ####
-                          NumFilesPerFileSink: 1
-                          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-#### A masked pattern was here ####
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              properties:
-                                columns _col0,_col1,_col2,_col3
-                                columns.types string:string:string:string
-                                escape.delim \
-                                hive.serialization.extend.nesting.levels true
-                                serialization.format 1
-                                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                          TotalFiles: 1
-                          GatherStats: false
-                          MultiFileSpray: false
-            Local Work:
-              Map Reduce Local Work
+                      tag: 1
+                      value expressions: key (type: string)
+                      auto parallelism: false
             Path -> Alias:
-              -mr-10004default.src{} [s1]
+              -mr-10003default.src{} [s2]
             Path -> Partition:
-              -mr-10004default.src{} 
+              -mr-10003default.src{} 
                 Partition
                   base file name: src
                   input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
@@ -1971,7 +1915,43 @@ STAGE PLANS:
                     name: default.src
                   name: default.src
             Truncated Path -> Alias:
-              -mr-10004default.src{} [s1]
+              -mr-10003default.src{} [s2]
+        Reducer 2 
+            Needs Tagging: true
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0} {KEY.reducesinkkey0}
+                  1 {VALUE._col0} {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col1, _col5, _col6
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+                  outputColumnNames: _col0, _col1, _col2, _col3
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+#### A masked pattern was here ####
+                    NumFilesPerFileSink: 1
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+#### A masked pattern was here ####
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        properties:
+                          columns _col0,_col1,_col2,_col3
+                          columns.types string:string:string:string
+                          escape.delim \
+                          hive.serialization.extend.nesting.levels true
+                          serialization.format 1
+                          serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    TotalFiles: 1
+                    GatherStats: false
+                    MultiFileSpray: false
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join0.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join0.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join0.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join0.q.out Tue Dec  2 19:57:10 2014
@@ -19,16 +19,18 @@ SELECT src1.key as k1, src1.value as v1,
   SORT BY k1, v1, k2, v2
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+        Reducer 3 <- Reducer 2 (SORT, 4)
 #### A masked pattern was here ####
       Vertices:
-        Map 3 
+        Map 1 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -40,23 +42,11 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
-                      Spark HashTable Sink Operator
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0} {_col1}
-                        keys:
-                          0 
-                          1 
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-      Edges:
-        Reducer 2 <- Map 1 (SORT, 4)
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: string), _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -68,27 +58,26 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0} {_col1}
-                        keys:
-                          0 
-                          1 
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        input vertices:
-                          1 Map 3
-                        Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
-                          sort order: ++++
-                          Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
-            Local Work:
-              Map Reduce Local Work
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: string), _col1 (type: string)
         Reducer 2 
             Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0} {VALUE._col1}
+                  1 {VALUE._col0} {VALUE._col1}
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+                  sort order: ++++
+                  Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+        Reducer 3 
+            Reduce Operator Tree:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join1.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_join1.q.out Tue Dec  2 19:57:10 2014
@@ -19,73 +19,66 @@ FROM src src1 JOIN src src2 ON (src1.key
 INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-3 is a root stage
-  Stage-1 depends on stages: Stage-3
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-3
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL SORT, 4)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
             Map Operator Tree:
                 TableScan
-                  alias: src2
+                  alias: src1
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {key}
-                        1 {value}
-                      keys:
-                        0 key (type: string)
-                        1 key (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+                    Reduce Output Operator
+                      key expressions: key (type: string)
+                      sort order: +
+                      Map-reduce partition columns: key (type: string)
+                      Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+        Map 3 
             Map Operator Tree:
                 TableScan
-                  alias: src1
+                  alias: src2
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-                    Map Join Operator
-                      condition map:
-                           Inner Join 0 to 1
-                      condition expressions:
-                        0 {key}
-                        1 {value}
-                      keys:
-                        0 key (type: string)
-                        1 key (type: string)
-                      outputColumnNames: _col0, _col6
-                      input vertices:
-                        1 Map 2
-                      Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                      Select Operator
-                        expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                        File Output Operator
-                          compressed: false
-                          Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.dest_j1
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      key expressions: key (type: string)
+                      sort order: +
+                      Map-reduce partition columns: key (type: string)
+                      Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                      value expressions: value (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0}
+                  1 {VALUE._col0}
+                outputColumnNames: _col0, _col6
+                Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: UDFToInteger(_col0) (type: int), _col6 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        name: default.dest_j1
 
   Stage: Stage-0
     Move Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/pcr.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/pcr.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/pcr.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/pcr.q.out Tue Dec  2 19:57:10 2014
@@ -2746,35 +2746,35 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
-        Map 3 
+        Map 1 
             Map Operator Tree:
                 TableScan
-                  alias: t2
+                  alias: t1
                   Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {key} {value}
-                        1 {value}
-                      keys:
-                        0 key (type: int)
-                        1 key (type: int)
-                      Position of Big Table: 0
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      key expressions: key (type: int)
+                      sort order: +
+                      Map-reduce partition columns: key (type: int)
+                      Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
+                      tag: 0
+                      value expressions: value (type: string)
+                      auto parallelism: false
             Path -> Alias:
 #### A masked pattern was here ####
             Path -> Partition:
@@ -2824,51 +2824,25 @@ STAGE PLANS:
                     name: default.pcr_t1
                   name: default.pcr_t1
             Truncated Path -> Alias:
-              /pcr_t1/ds=2000-04-08 [t2]
-
-  Stage: Stage-1
-    Spark
-      Edges:
-        Reducer 2 <- Map 1 (SORT, 1)
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+              /pcr_t1/ds=2000-04-08 [t1]
+        Map 4 
             Map Operator Tree:
                 TableScan
-                  alias: t1
+                  alias: t2
                   Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
-                    Map Join Operator
-                      condition map:
-                           Inner Join 0 to 1
-                      condition expressions:
-                        0 {key} {value}
-                        1 {key} {value}
-                      keys:
-                        0 key (type: int)
-                        1 key (type: int)
-                      outputColumnNames: _col0, _col1, _col6, _col7
-                      input vertices:
-                        1 Map 3
-                      Position of Big Table: 0
-                      Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
-                      Select Operator
-                        expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string)
-                        outputColumnNames: _col0, _col1, _col3, _col4
-                        Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: int)
-                          sort order: +
-                          Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
-                          tag: -1
-                          value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string)
-                          auto parallelism: false
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      key expressions: key (type: int)
+                      sort order: +
+                      Map-reduce partition columns: key (type: int)
+                      Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
+                      tag: 1
+                      value expressions: value (type: string)
+                      auto parallelism: false
             Path -> Alias:
 #### A masked pattern was here ####
             Path -> Partition:
@@ -2918,8 +2892,30 @@ STAGE PLANS:
                     name: default.pcr_t1
                   name: default.pcr_t1
             Truncated Path -> Alias:
-              /pcr_t1/ds=2000-04-08 [t1]
+              /pcr_t1/ds=2000-04-08 [t2]
         Reducer 2 
+            Needs Tagging: true
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col6, _col7
+                Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string)
+                  outputColumnNames: _col0, _col1, _col3, _col4
+                  Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: int)
+                    sort order: +
+                    Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+                    tag: -1
+                    value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string)
+                    auto parallelism: false
+        Reducer 3 
             Needs Tagging: false
             Reduce Operator Tree:
               Select Operator
@@ -3058,45 +3054,45 @@ TOK_QUERY
 
 
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 3 <- Reducer 2 (SORT, 1)
 #### A masked pattern was here ####
       Vertices:
-        Map 3 
+        Map 1 
             Map Operator Tree:
                 TableScan
-                  alias: t2
+                  alias: t1
                   Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {key} {value}
-                        1 {value}
-                      keys:
-                        0 key (type: int)
-                        1 key (type: int)
-                      Position of Big Table: 0
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      key expressions: key (type: int)
+                      sort order: +
+                      Map-reduce partition columns: key (type: int)
+                      Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
+                      tag: 0
+                      value expressions: value (type: string)
+                      auto parallelism: false
             Path -> Alias:
 #### A masked pattern was here ####
             Path -> Partition:
 #### A masked pattern was here ####
                 Partition
-                  base file name: ds=2000-04-09
+                  base file name: ds=2000-04-08
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                   partition values:
-                    ds 2000-04-09
+                    ds 2000-04-08
                   properties:
                     COLUMN_STATS_ACCURATE true
                     bucket_count -1
@@ -3136,61 +3132,35 @@ STAGE PLANS:
                     name: default.pcr_t1
                   name: default.pcr_t1
             Truncated Path -> Alias:
-              /pcr_t1/ds=2000-04-09 [t2]
-
-  Stage: Stage-1
-    Spark
-      Edges:
-        Reducer 2 <- Map 1 (SORT, 1)
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+              /pcr_t1/ds=2000-04-08 [t1]
+        Map 4 
             Map Operator Tree:
                 TableScan
-                  alias: t1
+                  alias: t2
                   Statistics: Num rows: 20 Data size: 160 Basic stats: COMPLETE Column stats: NONE
                   GatherStats: false
                   Filter Operator
                     isSamplingPred: false
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
-                    Map Join Operator
-                      condition map:
-                           Inner Join 0 to 1
-                      condition expressions:
-                        0 {key} {value}
-                        1 {key} {value}
-                      keys:
-                        0 key (type: int)
-                        1 key (type: int)
-                      outputColumnNames: _col0, _col1, _col6, _col7
-                      input vertices:
-                        1 Map 3
-                      Position of Big Table: 0
-                      Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
-                      Select Operator
-                        expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string)
-                        outputColumnNames: _col0, _col1, _col3, _col4
-                        Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: int)
-                          sort order: +
-                          Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
-                          tag: -1
-                          value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string)
-                          auto parallelism: false
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      key expressions: key (type: int)
+                      sort order: +
+                      Map-reduce partition columns: key (type: int)
+                      Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: NONE
+                      tag: 1
+                      value expressions: value (type: string)
+                      auto parallelism: false
             Path -> Alias:
 #### A masked pattern was here ####
             Path -> Partition:
 #### A masked pattern was here ####
                 Partition
-                  base file name: ds=2000-04-08
+                  base file name: ds=2000-04-09
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                   partition values:
-                    ds 2000-04-08
+                    ds 2000-04-09
                   properties:
                     COLUMN_STATS_ACCURATE true
                     bucket_count -1
@@ -3230,8 +3200,30 @@ STAGE PLANS:
                     name: default.pcr_t1
                   name: default.pcr_t1
             Truncated Path -> Alias:
-              /pcr_t1/ds=2000-04-08 [t1]
+              /pcr_t1/ds=2000-04-09 [t2]
         Reducer 2 
+            Needs Tagging: true
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col6, _col7
+                Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: int), _col1 (type: string), _col6 (type: int), _col7 (type: string)
+                  outputColumnNames: _col0, _col1, _col3, _col4
+                  Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: int)
+                    sort order: +
+                    Statistics: Num rows: 11 Data size: 88 Basic stats: COMPLETE Column stats: NONE
+                    tag: -1
+                    value expressions: _col1 (type: string), _col3 (type: int), _col4 (type: string)
+                    auto parallelism: false
+        Reducer 3 
             Needs Tagging: false
             Reduce Operator Tree:
               Select Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_gby_join.q.out Tue Dec  2 19:57:10 2014
@@ -19,13 +19,15 @@ WHERE src1.c1 > '20' AND (src1.c2 < 'val
 GROUP BY src1.c1
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+        Reducer 3 <- Reducer 2 (GROUP, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -43,23 +45,13 @@ STAGE PLANS:
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
-                        Spark HashTable Sink Operator
-                          condition expressions:
-                            0 {_col1}
-                            1 {_col0}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-      Edges:
-        Reducer 3 <- Map 2 (GROUP, 1)
-#### A masked pattern was here ####
-      Vertices:
-        Map 2 
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -74,40 +66,40 @@ STAGE PLANS:
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Inner Join 0 to 1
-                          condition expressions:
-                            0 {_col0} {_col1}
-                            1 {_col0}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-                          outputColumnNames: _col0, _col1, _col2
-                          input vertices:
-                            0 Map 1
-                          Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
-                          Filter Operator
-                            predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
-                            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                            Select Operator
-                              expressions: _col0 (type: string)
-                              outputColumnNames: _col0
-                              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                              Group By Operator
-                                aggregations: count(1)
-                                keys: _col0 (type: string)
-                                mode: hash
-                                outputColumnNames: _col0, _col1
-                                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                                Reduce Output Operator
-                                  key expressions: _col0 (type: string)
-                                  sort order: +
-                                  Map-reduce partition columns: _col0 (type: string)
-                                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                                  value expressions: _col1 (type: bigint)
-            Local Work:
-              Map Reduce Local Work
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(1)
+                      keys: _col0 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                        value expressions: _col1 (type: bigint)
         Reducer 3 
             Reduce Operator Tree:
               Group By Operator
@@ -155,13 +147,15 @@ WHERE src1.c1 > '20' AND (src1.c2 < 'val
 GROUP BY src1.c1
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+        Reducer 3 <- Reducer 2 (GROUP, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -176,23 +170,13 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                      Spark HashTable Sink Operator
-                        condition expressions:
-                          0 {_col1}
-                          1 {_col0}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-      Edges:
-        Reducer 3 <- Map 2 (GROUP, 1)
-#### A masked pattern was here ####
-      Vertices:
-        Map 2 
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -204,40 +188,40 @@ STAGE PLANS:
                       expressions: key (type: string)
                       outputColumnNames: _col0
                       Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                        outputColumnNames: _col0, _col1, _col2
-                        input vertices:
-                          0 Map 1
-                        Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE
-                        Filter Operator
-                          predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
-                          Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                          Select Operator
-                            expressions: _col0 (type: string)
-                            outputColumnNames: _col0
-                            Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                            Group By Operator
-                              aggregations: count(1)
-                              keys: _col0 (type: string)
-                              mode: hash
-                              outputColumnNames: _col0, _col1
-                              Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                              Reduce Output Operator
-                                key expressions: _col0 (type: string)
-                                sort order: +
-                                Map-reduce partition columns: _col0 (type: string)
-                                Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                                value expressions: _col1 (type: bigint)
-            Local Work:
-              Map Reduce Local Work
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+                  Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count(1)
+                      keys: _col0 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: bigint)
         Reducer 3 
             Reduce Operator Tree:
               Group By Operator