You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/08/06 00:31:32 UTC
svn commit: r1510788 [5/10] - in /hive/branches/tez: ./
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/
eclipse-templates/ hcatalog/storage-handlers/hbase/...
Modified: hive/branches/tez/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out Mon Aug 5 22:31:28 2013
@@ -1460,8 +1460,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-5 is a root stage
- Stage-4 depends on stages: Stage-5
- Stage-2 depends on stages: Stage-4
+ Stage-2 depends on stages: Stage-5
Stage-0 is a root stage
STAGE PLANS:
@@ -1490,7 +1489,7 @@ STAGE PLANS:
1 [Column[_col0]]
Position of Big Table: 0
- Stage: Stage-4
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
subq1:a
@@ -1519,25 +1518,14 @@ STAGE PLANS:
bucketGroup: false
mode: hash
outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
Local Work:
Map Reduce Local Work
-
- Stage: Stage-2
- Map Reduce
- Alias -> Map Operator Tree:
-#### A masked pattern was here ####
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
Reduce Operator Tree:
Group By Operator
aggregations:
Modified: hive/branches/tez/ql/src/test/results/clientpositive/avro_nullable_fields.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/avro_nullable_fields.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/avro_nullable_fields.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/avro_nullable_fields.q.out Mon Aug 5 22:31:28 2013
@@ -12,8 +12,8 @@ CREATE TABLE test_serializer(string1 STR
struct1 STRUCT<sint:INT,sboolean:BOOLEAN,sstring:STRING>,
enum1 STRING,
nullableint INT,
- bytes1 ARRAY<TINYINT>,
- fixed1 ARRAY<TINYINT>)
+ bytes1 BINARY,
+ fixed1 BINARY)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY ':' MAP KEYS TERMINATED BY '#' LINES TERMINATED BY '\n'
STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
@@ -31,8 +31,8 @@ CREATE TABLE test_serializer(string1 STR
struct1 STRUCT<sint:INT,sboolean:BOOLEAN,sstring:STRING>,
enum1 STRING,
nullableint INT,
- bytes1 ARRAY<TINYINT>,
- fixed1 ARRAY<TINYINT>)
+ bytes1 BINARY,
+ fixed1 BINARY)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY ':' MAP KEYS TERMINATED BY '#' LINES TERMINATED BY '\n'
STORED AS TEXTFILE
POSTHOOK: type: CREATETABLE
@@ -124,10 +124,10 @@ POSTHOOK: Input: default@test_serializer
POSTHOOK: Output: default@as_avro
POSTHOOK: Lineage: as_avro.bigint1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:bigint1, type:bigint, comment:null), ]
POSTHOOK: Lineage: as_avro.boolean1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:boolean1, type:boolean, comment:null), ]
-POSTHOOK: Lineage: as_avro.bytes1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:bytes1, type:array<tinyint>, comment:null), ]
+POSTHOOK: Lineage: as_avro.bytes1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:bytes1, type:binary, comment:null), ]
POSTHOOK: Lineage: as_avro.double1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:double1, type:double, comment:null), ]
POSTHOOK: Lineage: as_avro.enum1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:enum1, type:string, comment:null), ]
-POSTHOOK: Lineage: as_avro.fixed1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:fixed1, type:array<tinyint>, comment:null), ]
+POSTHOOK: Lineage: as_avro.fixed1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:fixed1, type:binary, comment:null), ]
POSTHOOK: Lineage: as_avro.float1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:float1, type:float, comment:null), ]
POSTHOOK: Lineage: as_avro.int1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:int1, type:int, comment:null), ]
POSTHOOK: Lineage: as_avro.list1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:list1, type:array<string>, comment:null), ]
@@ -147,10 +147,10 @@ POSTHOOK: Input: default@as_avro
#### A masked pattern was here ####
POSTHOOK: Lineage: as_avro.bigint1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:bigint1, type:bigint, comment:null), ]
POSTHOOK: Lineage: as_avro.boolean1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:boolean1, type:boolean, comment:null), ]
-POSTHOOK: Lineage: as_avro.bytes1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:bytes1, type:array<tinyint>, comment:null), ]
+POSTHOOK: Lineage: as_avro.bytes1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:bytes1, type:binary, comment:null), ]
POSTHOOK: Lineage: as_avro.double1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:double1, type:double, comment:null), ]
POSTHOOK: Lineage: as_avro.enum1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:enum1, type:string, comment:null), ]
-POSTHOOK: Lineage: as_avro.fixed1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:fixed1, type:array<tinyint>, comment:null), ]
+POSTHOOK: Lineage: as_avro.fixed1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:fixed1, type:binary, comment:null), ]
POSTHOOK: Lineage: as_avro.float1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:float1, type:float, comment:null), ]
POSTHOOK: Lineage: as_avro.int1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:int1, type:int, comment:null), ]
POSTHOOK: Lineage: as_avro.list1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:list1, type:array<string>, comment:null), ]
@@ -160,21 +160,21 @@ POSTHOOK: Lineage: as_avro.smallint1 EXP
POSTHOOK: Lineage: as_avro.string1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:string1, type:string, comment:null), ]
POSTHOOK: Lineage: as_avro.struct1 SIMPLE [(test_serializer)test_serializer.FieldSchema(name:struct1, type:struct<sint:int,sboolean:boolean,sstring:string>, comment:null), ]
POSTHOOK: Lineage: as_avro.tinyint1 EXPRESSION [(test_serializer)test_serializer.FieldSchema(name:tinyint1, type:tinyint, comment:null), ]
-why hello there 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-another record 98 4 101 9999999 false 99.89 9.0E-8 ["beta"] {"Earth":101} {"sint":1134,"sboolean":false,"sstring":"wazzup"} RED NULL [6,7,8,9,10] [54,55,56]
-third record 45 5 102 999999999 true 89.99 9.0E-14 ["alpha","gamma"] {"Earth":237,"Bob":723} {"sint":102,"sboolean":false,"sstring":"BNL"} GREEN NULL [11,12,13] [57,58,59]
-NULL 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string NULL 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 NULL 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 NULL 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 NULL true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 NULL 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true NULL 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 NULL ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 NULL {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] NULL {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} NULL BLUE 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} NULL 72 [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE NULL [0,1,2,3,4,5] [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 NULL [50,51,53]
-string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 [0,1,2,3,4,5] NULL
+why hello there 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+another record 98 4 101 9999999 false 99.89 9.0E-8 ["beta"] {"Earth":101} {"sint":1134,"sboolean":false,"sstring":"wazzup"} RED NULL ef
+third record 45 5 102 999999999 true 89.99 9.0E-14 ["alpha","gamma"] {"Earth":237,"Bob":723} {"sint":102,"sboolean":false,"sstring":"BNL"} GREEN NULL hi
+NULL 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string NULL 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 NULL 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 NULL 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 NULL true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 1412341 NULL 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 1412341 true NULL 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 1412341 true 42.43 NULL ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 1412341 true 42.43 85.23423424 NULL {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] NULL {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 bc
+string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} NULL BLUE 72 bc
+string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} NULL 72 bc
+string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE NULL bc
+string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 NULL bc
+string 42 3 100 1412341 true 42.43 85.23423424 ["alpha","beta","gamma"] {"Earth":42,"Bob":31,"Control":86} {"sint":17,"sboolean":true,"sstring":"Abe Linkedin"} BLUE 72 NULL
Modified: hive/branches/tez/ql/src/test/results/clientpositive/avro_schema_literal.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/avro_schema_literal.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/avro_schema_literal.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/avro_schema_literal.q.out Mon Aug 5 22:31:28 2013
@@ -77,5 +77,5 @@ struct1 struct<sint:int,sbo
union1 uniontype<float,boolean,string> from deserializer
enum1 string from deserializer
nullableint int from deserializer
-bytes1 array<tinyint> from deserializer
-fixed1 array<tinyint> from deserializer
+bytes1 binary from deserializer
+fixed1 binary from deserializer
Modified: hive/branches/tez/ql/src/test/results/clientpositive/ba_table_udfs.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/ba_table_udfs.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/ba_table_udfs.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/ba_table_udfs.q.out Mon Aug 5 22:31:28 2013
@@ -2,137 +2,179 @@ PREHOOK: query: USE default
PREHOOK: type: SWITCHDATABASE
POSTHOOK: query: USE default
POSTHOOK: type: SWITCHDATABASE
-PREHOOK: query: -- this query tests all the udfs provided to work with binary types
-
+PREHOOK: query: CREATE TABLE dest1(bytes1 BINARY,
+ bytes2 BINARY)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(bytes1 BINARY,
+ bytes2 BINARY)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1
SELECT
- key,
- value,
- LENGTH(CAST(src.key AS BINARY)),
- LENGTH(CAST(src.value AS BINARY)),
- CONCAT(CAST(src.key AS BINARY), CAST(src.value AS BINARY)),
- SUBSTR(CAST(src.value AS BINARY), 1, 4),
- SUBSTR(CAST(src.value AS BINARY), 3),
- SUBSTR(CAST(src.value AS BINARY), -4, 3)
-FROM src
+ CAST(key AS BINARY),
+ CAST(value AS BINARY)
ORDER BY value
LIMIT 100
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-#### A masked pattern was here ####
-POSTHOOK: query: -- this query tests all the udfs provided to work with binary types
-
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1
SELECT
- key,
- value,
- LENGTH(CAST(src.key AS BINARY)),
- LENGTH(CAST(src.value AS BINARY)),
- CONCAT(CAST(src.key AS BINARY), CAST(src.value AS BINARY)),
- SUBSTR(CAST(src.value AS BINARY), 1, 4),
- SUBSTR(CAST(src.value AS BINARY), 3),
- SUBSTR(CAST(src.value AS BINARY), -4, 3)
-FROM src
+ CAST(key AS BINARY),
+ CAST(value AS BINARY)
ORDER BY value
LIMIT 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.bytes1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.bytes2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: --Add in a null row for good measure
+INSERT INTO TABLE dest1 SELECT NULL, NULL FROM dest1 LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: default@dest1
+POSTHOOK: query: --Add in a null row for good measure
+INSERT INTO TABLE dest1 SELECT NULL, NULL FROM dest1 LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.bytes1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.bytes1 EXPRESSION []
+POSTHOOK: Lineage: dest1.bytes2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.bytes2 EXPRESSION []
+PREHOOK: query: -- this query tests all the udfs provided to work with binary types
+
+SELECT
+ bytes1,
+ bytes2,
+ LENGTH(bytes1),
+ CONCAT(bytes1, bytes2),
+ SUBSTR(bytes2, 1, 4),
+ SUBSTR(bytes2, 3),
+ SUBSTR(bytes2, -4, 3)
+FROM dest1
+ORDER BY bytes2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: -- this query tests all the udfs provided to work with binary types
+
+SELECT
+ bytes1,
+ bytes2,
+ LENGTH(bytes1),
+ CONCAT(bytes1, bytes2),
+ SUBSTR(bytes2, 1, 4),
+ SUBSTR(bytes2, 3),
+ SUBSTR(bytes2, -4, 3)
+FROM dest1
+ORDER BY bytes2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
-0 val_0 1 5 0val_0 val_ l_0 al_
-0 val_0 1 5 0val_0 val_ l_0 al_
-0 val_0 1 5 0val_0 val_ l_0 al_
-10 val_10 2 6 10val_10 val_ l_10 l_1
-100 val_100 3 7 100val_100 val_ l_100 _10
-100 val_100 3 7 100val_100 val_ l_100 _10
-103 val_103 3 7 103val_103 val_ l_103 _10
-103 val_103 3 7 103val_103 val_ l_103 _10
-104 val_104 3 7 104val_104 val_ l_104 _10
-104 val_104 3 7 104val_104 val_ l_104 _10
-105 val_105 3 7 105val_105 val_ l_105 _10
-11 val_11 2 6 11val_11 val_ l_11 l_1
-111 val_111 3 7 111val_111 val_ l_111 _11
-113 val_113 3 7 113val_113 val_ l_113 _11
-113 val_113 3 7 113val_113 val_ l_113 _11
-114 val_114 3 7 114val_114 val_ l_114 _11
-116 val_116 3 7 116val_116 val_ l_116 _11
-118 val_118 3 7 118val_118 val_ l_118 _11
-118 val_118 3 7 118val_118 val_ l_118 _11
-119 val_119 3 7 119val_119 val_ l_119 _11
-119 val_119 3 7 119val_119 val_ l_119 _11
-119 val_119 3 7 119val_119 val_ l_119 _11
-12 val_12 2 6 12val_12 val_ l_12 l_1
-12 val_12 2 6 12val_12 val_ l_12 l_1
-120 val_120 3 7 120val_120 val_ l_120 _12
-120 val_120 3 7 120val_120 val_ l_120 _12
-125 val_125 3 7 125val_125 val_ l_125 _12
-125 val_125 3 7 125val_125 val_ l_125 _12
-126 val_126 3 7 126val_126 val_ l_126 _12
-128 val_128 3 7 128val_128 val_ l_128 _12
-128 val_128 3 7 128val_128 val_ l_128 _12
-128 val_128 3 7 128val_128 val_ l_128 _12
-129 val_129 3 7 129val_129 val_ l_129 _12
-129 val_129 3 7 129val_129 val_ l_129 _12
-131 val_131 3 7 131val_131 val_ l_131 _13
-133 val_133 3 7 133val_133 val_ l_133 _13
-134 val_134 3 7 134val_134 val_ l_134 _13
-134 val_134 3 7 134val_134 val_ l_134 _13
-136 val_136 3 7 136val_136 val_ l_136 _13
-137 val_137 3 7 137val_137 val_ l_137 _13
-137 val_137 3 7 137val_137 val_ l_137 _13
-138 val_138 3 7 138val_138 val_ l_138 _13
-138 val_138 3 7 138val_138 val_ l_138 _13
-138 val_138 3 7 138val_138 val_ l_138 _13
-138 val_138 3 7 138val_138 val_ l_138 _13
-143 val_143 3 7 143val_143 val_ l_143 _14
-145 val_145 3 7 145val_145 val_ l_145 _14
-146 val_146 3 7 146val_146 val_ l_146 _14
-146 val_146 3 7 146val_146 val_ l_146 _14
-149 val_149 3 7 149val_149 val_ l_149 _14
-149 val_149 3 7 149val_149 val_ l_149 _14
-15 val_15 2 6 15val_15 val_ l_15 l_1
-15 val_15 2 6 15val_15 val_ l_15 l_1
-150 val_150 3 7 150val_150 val_ l_150 _15
-152 val_152 3 7 152val_152 val_ l_152 _15
-152 val_152 3 7 152val_152 val_ l_152 _15
-153 val_153 3 7 153val_153 val_ l_153 _15
-155 val_155 3 7 155val_155 val_ l_155 _15
-156 val_156 3 7 156val_156 val_ l_156 _15
-157 val_157 3 7 157val_157 val_ l_157 _15
-158 val_158 3 7 158val_158 val_ l_158 _15
-160 val_160 3 7 160val_160 val_ l_160 _16
-162 val_162 3 7 162val_162 val_ l_162 _16
-163 val_163 3 7 163val_163 val_ l_163 _16
-164 val_164 3 7 164val_164 val_ l_164 _16
-164 val_164 3 7 164val_164 val_ l_164 _16
-165 val_165 3 7 165val_165 val_ l_165 _16
-165 val_165 3 7 165val_165 val_ l_165 _16
-166 val_166 3 7 166val_166 val_ l_166 _16
-167 val_167 3 7 167val_167 val_ l_167 _16
-167 val_167 3 7 167val_167 val_ l_167 _16
-167 val_167 3 7 167val_167 val_ l_167 _16
-168 val_168 3 7 168val_168 val_ l_168 _16
-169 val_169 3 7 169val_169 val_ l_169 _16
-169 val_169 3 7 169val_169 val_ l_169 _16
-169 val_169 3 7 169val_169 val_ l_169 _16
-169 val_169 3 7 169val_169 val_ l_169 _16
-17 val_17 2 6 17val_17 val_ l_17 l_1
-170 val_170 3 7 170val_170 val_ l_170 _17
-172 val_172 3 7 172val_172 val_ l_172 _17
-172 val_172 3 7 172val_172 val_ l_172 _17
-174 val_174 3 7 174val_174 val_ l_174 _17
-174 val_174 3 7 174val_174 val_ l_174 _17
-175 val_175 3 7 175val_175 val_ l_175 _17
-175 val_175 3 7 175val_175 val_ l_175 _17
-176 val_176 3 7 176val_176 val_ l_176 _17
-176 val_176 3 7 176val_176 val_ l_176 _17
-177 val_177 3 7 177val_177 val_ l_177 _17
-178 val_178 3 7 178val_178 val_ l_178 _17
-179 val_179 3 7 179val_179 val_ l_179 _17
-179 val_179 3 7 179val_179 val_ l_179 _17
-18 val_18 2 6 18val_18 val_ l_18 l_1
-18 val_18 2 6 18val_18 val_ l_18 l_1
-180 val_180 3 7 180val_180 val_ l_180 _18
-181 val_181 3 7 181val_181 val_ l_181 _18
-183 val_183 3 7 183val_183 val_ l_183 _18
-186 val_186 3 7 186val_186 val_ l_186 _18
-187 val_187 3 7 187val_187 val_ l_187 _18
-187 val_187 3 7 187val_187 val_ l_187 _18
-187 val_187 3 7 187val_187 val_ l_187 _18
+POSTHOOK: Lineage: dest1.bytes1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.bytes1 EXPRESSION []
+POSTHOOK: Lineage: dest1.bytes2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.bytes2 EXPRESSION []
+NULL NULL NULL NULL NULL NULL NULL
+0 val_0 1 0val_0 val_ l_0 al_
+0 val_0 1 0val_0 val_ l_0 al_
+0 val_0 1 0val_0 val_ l_0 al_
+10 val_10 2 10val_10 val_ l_10 l_1
+100 val_100 3 100val_100 val_ l_100 _10
+100 val_100 3 100val_100 val_ l_100 _10
+103 val_103 3 103val_103 val_ l_103 _10
+103 val_103 3 103val_103 val_ l_103 _10
+104 val_104 3 104val_104 val_ l_104 _10
+104 val_104 3 104val_104 val_ l_104 _10
+105 val_105 3 105val_105 val_ l_105 _10
+11 val_11 2 11val_11 val_ l_11 l_1
+111 val_111 3 111val_111 val_ l_111 _11
+113 val_113 3 113val_113 val_ l_113 _11
+113 val_113 3 113val_113 val_ l_113 _11
+114 val_114 3 114val_114 val_ l_114 _11
+116 val_116 3 116val_116 val_ l_116 _11
+118 val_118 3 118val_118 val_ l_118 _11
+118 val_118 3 118val_118 val_ l_118 _11
+119 val_119 3 119val_119 val_ l_119 _11
+119 val_119 3 119val_119 val_ l_119 _11
+119 val_119 3 119val_119 val_ l_119 _11
+12 val_12 2 12val_12 val_ l_12 l_1
+12 val_12 2 12val_12 val_ l_12 l_1
+120 val_120 3 120val_120 val_ l_120 _12
+120 val_120 3 120val_120 val_ l_120 _12
+125 val_125 3 125val_125 val_ l_125 _12
+125 val_125 3 125val_125 val_ l_125 _12
+126 val_126 3 126val_126 val_ l_126 _12
+128 val_128 3 128val_128 val_ l_128 _12
+128 val_128 3 128val_128 val_ l_128 _12
+128 val_128 3 128val_128 val_ l_128 _12
+129 val_129 3 129val_129 val_ l_129 _12
+129 val_129 3 129val_129 val_ l_129 _12
+131 val_131 3 131val_131 val_ l_131 _13
+133 val_133 3 133val_133 val_ l_133 _13
+134 val_134 3 134val_134 val_ l_134 _13
+134 val_134 3 134val_134 val_ l_134 _13
+136 val_136 3 136val_136 val_ l_136 _13
+137 val_137 3 137val_137 val_ l_137 _13
+137 val_137 3 137val_137 val_ l_137 _13
+138 val_138 3 138val_138 val_ l_138 _13
+138 val_138 3 138val_138 val_ l_138 _13
+138 val_138 3 138val_138 val_ l_138 _13
+138 val_138 3 138val_138 val_ l_138 _13
+143 val_143 3 143val_143 val_ l_143 _14
+145 val_145 3 145val_145 val_ l_145 _14
+146 val_146 3 146val_146 val_ l_146 _14
+146 val_146 3 146val_146 val_ l_146 _14
+149 val_149 3 149val_149 val_ l_149 _14
+149 val_149 3 149val_149 val_ l_149 _14
+15 val_15 2 15val_15 val_ l_15 l_1
+15 val_15 2 15val_15 val_ l_15 l_1
+150 val_150 3 150val_150 val_ l_150 _15
+152 val_152 3 152val_152 val_ l_152 _15
+152 val_152 3 152val_152 val_ l_152 _15
+153 val_153 3 153val_153 val_ l_153 _15
+155 val_155 3 155val_155 val_ l_155 _15
+156 val_156 3 156val_156 val_ l_156 _15
+157 val_157 3 157val_157 val_ l_157 _15
+158 val_158 3 158val_158 val_ l_158 _15
+160 val_160 3 160val_160 val_ l_160 _16
+162 val_162 3 162val_162 val_ l_162 _16
+163 val_163 3 163val_163 val_ l_163 _16
+164 val_164 3 164val_164 val_ l_164 _16
+164 val_164 3 164val_164 val_ l_164 _16
+165 val_165 3 165val_165 val_ l_165 _16
+165 val_165 3 165val_165 val_ l_165 _16
+166 val_166 3 166val_166 val_ l_166 _16
+167 val_167 3 167val_167 val_ l_167 _16
+167 val_167 3 167val_167 val_ l_167 _16
+167 val_167 3 167val_167 val_ l_167 _16
+168 val_168 3 168val_168 val_ l_168 _16
+169 val_169 3 169val_169 val_ l_169 _16
+169 val_169 3 169val_169 val_ l_169 _16
+169 val_169 3 169val_169 val_ l_169 _16
+169 val_169 3 169val_169 val_ l_169 _16
+17 val_17 2 17val_17 val_ l_17 l_1
+170 val_170 3 170val_170 val_ l_170 _17
+172 val_172 3 172val_172 val_ l_172 _17
+172 val_172 3 172val_172 val_ l_172 _17
+174 val_174 3 174val_174 val_ l_174 _17
+174 val_174 3 174val_174 val_ l_174 _17
+175 val_175 3 175val_175 val_ l_175 _17
+175 val_175 3 175val_175 val_ l_175 _17
+176 val_176 3 176val_176 val_ l_176 _17
+176 val_176 3 176val_176 val_ l_176 _17
+177 val_177 3 177val_177 val_ l_177 _17
+178 val_178 3 178val_178 val_ l_178 _17
+179 val_179 3 179val_179 val_ l_179 _17
+179 val_179 3 179val_179 val_ l_179 _17
+18 val_18 2 18val_18 val_ l_18 l_1
+18 val_18 2 18val_18 val_ l_18 l_1
+180 val_180 3 180val_180 val_ l_180 _18
+181 val_181 3 181val_181 val_ l_181 _18
+183 val_183 3 183val_183 val_ l_183 _18
+186 val_186 3 186val_186 val_ l_186 _18
+187 val_187 3 187val_187 val_ l_187 _18
+187 val_187 3 187val_187 val_ l_187 _18
+187 val_187 3 187val_187 val_ l_187 _18
Modified: hive/branches/tez/ql/src/test/results/clientpositive/combine2_win.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/combine2_win.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/combine2_win.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/combine2_win.q.out Mon Aug 5 22:31:28 2013
@@ -603,6 +603,7 @@ STAGE PLANS:
columns _col0
columns.types bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer1.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer1.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer1.q.out Mon Aug 5 22:31:28 2013
@@ -349,199 +349,6 @@ POSTHOOK: Input: default@src1
PREHOOK: query: -- Enable hive.auto.convert.join.
-- Correlation Optimizer will detect that the join will be converted to a Map-join,
-- so it will not try to optimize this query.
-EXPLAIN
-SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))
-FROM (SELECT x.key AS key, count(1) AS cnt
- FROM src1 x JOIN src y ON (x.key = y.key)
- GROUP BY x.key) tmp
-PREHOOK: type: QUERY
-POSTHOOK: query: -- Enable hive.auto.convert.join.
--- Correlation Optimizer will detect that the join will be converted to a Map-join,
--- so it will not try to optimize this query.
-EXPLAIN
-SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))
-FROM (SELECT x.key AS key, count(1) AS cnt
- FROM src1 x JOIN src y ON (x.key = y.key)
- GROUP BY x.key) tmp
-POSTHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
- (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key) key) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL x) key)))) tmp)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION SUM (TOK_FUNCTION HASH (. (TOK_TABLE_OR_COL tmp) key)))) (TOK_SELEXPR (TOK_FUNCTION SUM (TOK_FUNCTION HASH (. (TOK_TABLE_OR_COL tmp) cnt)))))))
-
-STAGE DEPENDENCIES:
- Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
- Stage-2 depends on stages: Stage-5
- Stage-3 depends on stages: Stage-2
- Stage-0 is a root stage
-
-STAGE PLANS:
- Stage: Stage-6
- Map Reduce Local Work
- Alias -> Map Local Tables:
- tmp:x
- Fetch Operator
- limit: -1
- Alias -> Map Local Operator Tree:
- tmp:x
- TableScan
- alias: x
- HashTable Sink Operator
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
-
- Stage: Stage-5
- Map Reduce
- Alias -> Map Operator Tree:
- tmp:y
- TableScan
- alias: y
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- outputColumnNames: _col0
- Position of Big Table: 1
- Select Operator
- expressions:
- expr: _col0
- type: string
- outputColumnNames: _col0
- Group By Operator
- aggregations:
- expr: count(1)
- bucketGroup: false
- keys:
- expr: _col0
- type: string
- mode: hash
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
- Stage: Stage-2
- Map Reduce
- Alias -> Map Operator Tree:
-#### A masked pattern was here ####
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: -1
- value expressions:
- expr: _col1
- type: bigint
- Reduce Operator Tree:
- Group By Operator
- aggregations:
- expr: count(VALUE._col0)
- bucketGroup: false
- keys:
- expr: KEY._col0
- type: string
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: bigint
- outputColumnNames: _col0, _col1
- Group By Operator
- aggregations:
- expr: sum(hash(_col0))
- expr: sum(hash(_col1))
- bucketGroup: false
- mode: hash
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-
- Stage: Stage-3
- Map Reduce
- Alias -> Map Operator Tree:
-#### A masked pattern was here ####
- Reduce Output Operator
- sort order:
- tag: -1
- value expressions:
- expr: _col0
- type: bigint
- expr: _col1
- type: bigint
- Reduce Operator Tree:
- Group By Operator
- aggregations:
- expr: sum(VALUE._col0)
- expr: sum(VALUE._col1)
- bucketGroup: false
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Select Operator
- expressions:
- expr: _col0
- type: bigint
- expr: _col1
- type: bigint
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
-
-
-PREHOOK: query: SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))
-FROM (SELECT x.key AS key, count(1) AS cnt
- FROM src1 x JOIN src y ON (x.key = y.key)
- GROUP BY x.key) tmp
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Input: default@src1
-#### A masked pattern was here ####
-POSTHOOK: query: SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))
-FROM (SELECT x.key AS key, count(1) AS cnt
- FROM src1 x JOIN src y ON (x.key = y.key)
- GROUP BY x.key) tmp
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Input: default@src1
-#### A masked pattern was here ####
-652447 37
-PREHOOK: query: -- Enable hive.auto.convert.join.
--- Correlation Optimizer will detect that the join will be converted to a Map-join,
--- so it will not try to optimize this query.
-- We should generate 1 MR job for subquery tmp.
EXPLAIN
SELECT SUM(HASH(tmp.key)), SUM(HASH(tmp.cnt))
@@ -564,8 +371,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
- Stage-3 depends on stages: Stage-5
+ Stage-2 depends on stages: Stage-6
+ Stage-3 depends on stages: Stage-2
Stage-0 is a root stage
STAGE PLANS:
@@ -589,7 +396,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 1
- Stage: Stage-5
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
tmp:y
Modified: hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer3.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer3.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer3.q.out Mon Aug 5 22:31:28 2013
@@ -598,11 +598,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-9 is a root stage
- Stage-7 depends on stages: Stage-9
- Stage-2 depends on stages: Stage-7, Stage-8
+ Stage-2 depends on stages: Stage-9
Stage-3 depends on stages: Stage-2
- Stage-10 is a root stage
- Stage-8 depends on stages: Stage-10
Stage-0 is a root stage
STAGE PLANS:
@@ -612,6 +609,9 @@ STAGE PLANS:
tmp:b:x
Fetch Operator
limit: -1
+ tmp:d:x
+ Fetch Operator
+ limit: -1
Alias -> Map Local Operator Tree:
tmp:b:x
TableScan
@@ -625,8 +625,20 @@ STAGE PLANS:
0 [Column[key]]
1 [Column[key]]
Position of Big Table: 1
+ tmp:d:x
+ TableScan
+ alias: x
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
- Stage: Stage-7
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
tmp:b:y
@@ -658,44 +670,54 @@ STAGE PLANS:
type: string
mode: hash
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
+ tmp:d:y
+ TableScan
+ alias: y
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0, _col1
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col1
+ type: string
Local Work:
Map Reduce Local Work
-
- Stage: Stage-2
- Map Reduce
- Alias -> Map Operator Tree:
- $INTNAME
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 0
- value expressions:
- expr: _col1
- type: bigint
- $INTNAME1
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 1
- value expressions:
- expr: _col1
- type: string
Reduce Operator Tree:
Demux Operator
Group By Operator
@@ -818,60 +840,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- Stage: Stage-10
- Map Reduce Local Work
- Alias -> Map Local Tables:
- tmp:d:x
- Fetch Operator
- limit: -1
- Alias -> Map Local Operator Tree:
- tmp:d:x
- TableScan
- alias: x
- HashTable Sink Operator
- condition expressions:
- 0 {key} {value}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
-
- Stage: Stage-8
- Map Reduce
- Alias -> Map Operator Tree:
- tmp:d:y
- TableScan
- alias: y
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {key} {value}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- outputColumnNames: _col0, _col1
- Position of Big Table: 1
- Select Operator
- expressions:
- expr: _col0
- type: string
- expr: _col1
- type: string
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
Stage: Stage-0
Fetch Operator
limit: -1
@@ -1482,11 +1450,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-9 is a root stage
- Stage-7 depends on stages: Stage-9
- Stage-2 depends on stages: Stage-7, Stage-8
+ Stage-2 depends on stages: Stage-9
Stage-3 depends on stages: Stage-2
- Stage-10 is a root stage
- Stage-8 depends on stages: Stage-10
Stage-0 is a root stage
STAGE PLANS:
@@ -1496,6 +1461,9 @@ STAGE PLANS:
tmp:b:x
Fetch Operator
limit: -1
+ tmp:d:x
+ Fetch Operator
+ limit: -1
Alias -> Map Local Operator Tree:
tmp:b:x
TableScan
@@ -1509,8 +1477,20 @@ STAGE PLANS:
0 [Column[key]]
1 [Column[key]]
Position of Big Table: 1
+ tmp:d:x
+ TableScan
+ alias: x
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 1
- Stage: Stage-7
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
tmp:b:y
@@ -1535,44 +1515,61 @@ STAGE PLANS:
expr: _col1
type: string
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: string
+ tmp:d:y
+ TableScan
+ alias: y
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0
+ Position of Big Table: 1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col1
+ type: bigint
Local Work:
Map Reduce Local Work
-
- Stage: Stage-2
- Map Reduce
- Alias -> Map Operator Tree:
- $INTNAME
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 0
- value expressions:
- expr: _col1
- type: string
- $INTNAME1
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 1
- value expressions:
- expr: _col1
- type: bigint
Reduce Operator Tree:
Demux Operator
Mux Operator
@@ -1695,67 +1692,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- Stage: Stage-10
- Map Reduce Local Work
- Alias -> Map Local Tables:
- tmp:d:x
- Fetch Operator
- limit: -1
- Alias -> Map Local Operator Tree:
- tmp:d:x
- TableScan
- alias: x
- HashTable Sink Operator
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 1
-
- Stage: Stage-8
- Map Reduce
- Alias -> Map Operator Tree:
- tmp:d:y
- TableScan
- alias: y
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- outputColumnNames: _col0
- Position of Big Table: 1
- Select Operator
- expressions:
- expr: _col0
- type: string
- outputColumnNames: _col0
- Group By Operator
- aggregations:
- expr: count(1)
- bucketGroup: false
- keys:
- expr: _col0
- type: string
- mode: hash
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
Stage: Stage-0
Fetch Operator
limit: -1
Modified: hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer4.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer4.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer4.q.out Mon Aug 5 22:31:28 2013
@@ -428,8 +428,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-7 is a root stage
- Stage-6 depends on stages: Stage-7
- Stage-3 depends on stages: Stage-6
+ Stage-2 depends on stages: Stage-7
+ Stage-3 depends on stages: Stage-2
Stage-0 is a root stage
STAGE PLANS:
@@ -472,7 +472,7 @@ STAGE PLANS:
2 [Column[key]]
Position of Big Table: 0
- Stage: Stage-6
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
tmp:x
Modified: hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer6.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer6.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer6.q.out Mon Aug 5 22:31:28 2013
@@ -661,21 +661,33 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-9 is a root stage
- Stage-7 depends on stages: Stage-9
- Stage-2 depends on stages: Stage-7, Stage-8
+ Stage-2 depends on stages: Stage-9
Stage-3 depends on stages: Stage-2
- Stage-10 is a root stage
- Stage-8 depends on stages: Stage-10
Stage-0 is a root stage
STAGE PLANS:
Stage: Stage-9
Map Reduce Local Work
Alias -> Map Local Tables:
+ xx:y
+ Fetch Operator
+ limit: -1
yy:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
+ xx:y
+ TableScan
+ alias: y
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
yy:y
TableScan
alias: y
@@ -689,9 +701,50 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-7
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
+ xx:x
+ TableScan
+ alias: x
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
yy:x
TableScan
alias: x
@@ -721,44 +774,20 @@ STAGE PLANS:
type: string
mode: hash
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ value expressions:
+ expr: _col1
+ type: bigint
Local Work:
Map Reduce Local Work
-
- Stage: Stage-2
- Map Reduce
- Alias -> Map Operator Tree:
- $INTNAME
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 1
- value expressions:
- expr: _col1
- type: bigint
- $INTNAME1
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 0
- value expressions:
- expr: _col1
- type: bigint
Reduce Operator Tree:
Demux Operator
Group By Operator
@@ -880,67 +909,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- Stage: Stage-10
- Map Reduce Local Work
- Alias -> Map Local Tables:
- xx:y
- Fetch Operator
- limit: -1
- Alias -> Map Local Operator Tree:
- xx:y
- TableScan
- alias: y
- HashTable Sink Operator
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 0
-
- Stage: Stage-8
- Map Reduce
- Alias -> Map Operator Tree:
- xx:x
- TableScan
- alias: x
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- outputColumnNames: _col0
- Position of Big Table: 0
- Select Operator
- expressions:
- expr: _col0
- type: string
- outputColumnNames: _col0
- Group By Operator
- aggregations:
- expr: count(1)
- bucketGroup: false
- keys:
- expr: _col0
- type: string
- mode: hash
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
Stage: Stage-0
Fetch Operator
limit: -1
@@ -5122,16 +5090,15 @@ ABSTRACT SYNTAX TREE:
(TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src1) x) (TOK_TABREF (TOK_TABNAME src1) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key) key) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL x) key)))) xx) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) x) (TOK_TABREF (TOK_TABNAME src) y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL x) value) value) (TOK_SELEXPR (TOK_FUNCTION count 1) cnt)) (TOK_GROUPBY (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL x) value)))) yy) (= (. (TOK_TABLE_OR_COL xx) key) (. (TOK_TABLE_OR_COL yy) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_
SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL xx) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL xx) cnt)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL yy) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL yy) value)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL yy) cnt))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL xx) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL xx) cnt)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL yy) key)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL yy) value)) (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL yy) cnt)))))
STAGE DEPENDENCIES:
- Stage-10 is a root stage
- Stage-8 depends on stages: Stage-10
- Stage-3 depends on stages: Stage-8, Stage-9
- Stage-4 depends on stages: Stage-3
Stage-11 is a root stage
- Stage-9 depends on stages: Stage-11
+ Stage-2 depends on stages: Stage-11
+ Stage-10 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-10
+ Stage-4 depends on stages: Stage-3
Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-10
+ Stage: Stage-11
Map Reduce Local Work
Alias -> Map Local Tables:
yy:y
@@ -5151,7 +5118,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-8
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
yy:x
@@ -5233,6 +5200,26 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ xx:y
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ xx:y
+ TableScan
+ alias: y
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ Position of Big Table: 0
+
Stage: Stage-3
Map Reduce
Alias -> Map Operator Tree:
@@ -5253,19 +5240,49 @@ STAGE PLANS:
type: string
expr: _col2
type: bigint
- $INTNAME1
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 0
- value expressions:
- expr: _col1
- type: bigint
+ xx:x
+ TableScan
+ alias: x
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[key]]
+ outputColumnNames: _col0
+ Position of Big Table: 0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Group By Operator
+ aggregations:
+ expr: count(1)
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Demux Operator
Group By Operator
@@ -5379,67 +5396,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- Stage: Stage-11
- Map Reduce Local Work
- Alias -> Map Local Tables:
- xx:y
- Fetch Operator
- limit: -1
- Alias -> Map Local Operator Tree:
- xx:y
- TableScan
- alias: y
- HashTable Sink Operator
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- Position of Big Table: 0
-
- Stage: Stage-9
- Map Reduce
- Alias -> Map Operator Tree:
- xx:x
- TableScan
- alias: x
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {key}
- 1
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[key]]
- outputColumnNames: _col0
- Position of Big Table: 0
- Select Operator
- expressions:
- expr: _col0
- type: string
- outputColumnNames: _col0
- Group By Operator
- aggregations:
- expr: count(1)
- bucketGroup: false
- keys:
- expr: _col0
- type: string
- mode: hash
- outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
Stage: Stage-0
Fetch Operator
limit: -1
Modified: hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer7.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer7.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/correlationoptimizer7.q.out Mon Aug 5 22:31:28 2013
@@ -19,11 +19,9 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-9 is a root stage
- Stage-7 depends on stages: Stage-9
- Stage-4 depends on stages: Stage-7
+ Stage-4 depends on stages: Stage-9
Stage-8 depends on stages: Stage-4
- Stage-6 depends on stages: Stage-8
- Stage-2 depends on stages: Stage-6
+ Stage-2 depends on stages: Stage-8
Stage-0 is a root stage
STAGE PLANS:
@@ -47,7 +45,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-7
+ Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
xx:x
@@ -79,31 +77,20 @@ STAGE PLANS:
type: string
mode: hash
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ value expressions:
+ expr: _col1
+ type: bigint
Local Work:
Map Reduce Local Work
-
- Stage: Stage-4
- Map Reduce
- Alias -> Map Operator Tree:
-#### A masked pattern was here ####
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: -1
- value expressions:
- expr: _col1
- type: bigint
Reduce Operator Tree:
Group By Operator
aggregations:
@@ -148,7 +135,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-6
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
$INTNAME
@@ -175,40 +162,29 @@ STAGE PLANS:
expr: _col3
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
+ sort order: ++++
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: bigint
+ expr: _col2
+ type: string
+ expr: _col3
+ type: string
Local Work:
Map Reduce Local Work
-
- Stage: Stage-2
- Map Reduce
- Alias -> Map Operator Tree:
-#### A masked pattern was here ####
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- expr: _col1
- type: bigint
- expr: _col2
- type: string
- expr: _col3
- type: string
- sort order: ++++
- tag: -1
- value expressions:
- expr: _col0
- type: string
- expr: _col1
- type: bigint
- expr: _col2
- type: string
- expr: _col3
- type: string
Reduce Operator Tree:
Extract
File Output Operator
@@ -279,8 +255,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
- Stage-1 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
@@ -305,7 +280,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-5
+ Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
xx:x
@@ -337,31 +312,18 @@ STAGE PLANS:
type: string
mode: hash
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
- Stage: Stage-1
- Map Reduce
- Alias -> Map Operator Tree:
- $INTNAME
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 0
- value expressions:
- expr: _col1
- type: bigint
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
yy
TableScan
alias: yy
@@ -379,6 +341,8 @@ STAGE PLANS:
type: string
expr: value
type: string
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Demux Operator
Group By Operator
@@ -527,13 +491,9 @@ POSTHOOK: Input: default@src1
PREHOOK: query: -- Without correlation optimizer, we will have 3 MR jobs.
-- The first one is a MapJoin and Aggregation (in the Reduce Phase).
-- The second one is another MapJoin. The third one is for ordering.
--- With the correlation optimizer, right now, we still have
--- 3 MR jobs. The first one is a MapJoin and the map-side aggregation (a map-only job).
--- The second one have the reduce-side aggregation and the second join.
--- The third one is for ordering.
--- Although we have turned on hive.optimize.mapjoin.mapreduce, that optimizer
--- can not handle the case that the MR job (the one which a map-only job will be merged in)
--- has multiple inputs. We should improve that optimizer.
+-- With the correlation optimizer, right now, we have
+-- 2 MR jobs. The first one will evaluate the sub-query xx and the join of
+-- xx and yy. The second one will do the ORDER BY.
EXPLAIN
SELECT xx.key, xx.cnt, yy.key, yy.value
FROM (SELECT x.key AS key, count(1) AS cnt
@@ -545,13 +505,9 @@ PREHOOK: type: QUERY
POSTHOOK: query: -- Without correlation optimizer, we will have 3 MR jobs.
-- The first one is a MapJoin and Aggregation (in the Reduce Phase).
-- The second one is another MapJoin. The third one is for ordering.
--- With the correlation optimizer, right now, we still have
--- 3 MR jobs. The first one is a MapJoin and the map-side aggregation (a map-only job).
--- The second one have the reduce-side aggregation and the second join.
--- The third one is for ordering.
--- Although we have turned on hive.optimize.mapjoin.mapreduce, that optimizer
--- can not handle the case that the MR job (the one which a map-only job will be merged in)
--- has multiple inputs. We should improve that optimizer.
+-- With the correlation optimizer, right now, we have
+-- 2 MR jobs. The first one will evaluate the sub-query xx and the join of
+-- xx and yy. The second one will do the ORDER BY.
EXPLAIN
SELECT xx.key, xx.cnt, yy.key, yy.value
FROM (SELECT x.key AS key, count(1) AS cnt
@@ -565,9 +521,9 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-9 is a root stage
- Stage-7 depends on stages: Stage-9
- Stage-8 depends on stages: Stage-7
- Stage-6 depends on stages: Stage-8
+ Stage-4 depends on stages: Stage-9
+ Stage-8 depends on stages: Stage-4
+ Stage-2 depends on stages: Stage-8
Stage-0 is a root stage
STAGE PLANS:
@@ -591,7 +547,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-7
+ Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
xx:x
@@ -681,7 +637,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-6
+ Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
$INTNAME
@@ -801,8 +757,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
- Stage-1 depends on stages: Stage-5
+ Stage-1 depends on stages: Stage-6
Stage-2 depends on stages: Stage-1
Stage-0 is a root stage
@@ -827,7 +782,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-5
+ Stage: Stage-1
Map Reduce
Alias -> Map Operator Tree:
xx:x
@@ -859,31 +814,18 @@ STAGE PLANS:
type: string
mode: hash
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- Local Work:
- Map Reduce Local Work
-
- Stage: Stage-1
- Map Reduce
- Alias -> Map Operator Tree:
- $INTNAME
- Reduce Output Operator
- key expressions:
- expr: _col0
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: _col0
- type: string
- tag: 0
- value expressions:
- expr: _col1
- type: bigint
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 0
+ value expressions:
+ expr: _col1
+ type: bigint
yy
TableScan
alias: yy
@@ -901,6 +843,8 @@ STAGE PLANS:
type: string
expr: value
type: string
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Demux Operator
Group By Operator
Modified: hive/branches/tez/ql/src/test/results/clientpositive/join28.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/join28.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/join28.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/join28.q.out Mon Aug 5 22:31:28 2013
@@ -28,8 +28,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-8 is a root stage
- Stage-7 depends on stages: Stage-8
- Stage-0 depends on stages: Stage-7
+ Stage-6 depends on stages: Stage-8
+ Stage-0 depends on stages: Stage-6
Stage-3 depends on stages: Stage-0
STAGE PLANS:
@@ -68,7 +68,7 @@ STAGE PLANS:
1 [Column[key]]
Position of Big Table: 0
- Stage: Stage-7
+ Stage: Stage-6
Map Reduce
Alias -> Map Operator Tree:
subq:y
Modified: hive/branches/tez/ql/src/test/results/clientpositive/join32.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/join32.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/join32.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/join32.q.out Mon Aug 5 22:31:28 2013
@@ -24,8 +24,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-7 is a root stage
- Stage-6 depends on stages: Stage-7
- Stage-0 depends on stages: Stage-6
+ Stage-5 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-5
Stage-2 depends on stages: Stage-0
STAGE PLANS:
@@ -66,7 +66,7 @@ STAGE PLANS:
1 [Column[value]]
Position of Big Table: 0
- Stage: Stage-6
+ Stage: Stage-5
Map Reduce
Alias -> Map Operator Tree:
y
@@ -219,6 +219,52 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src1
name: default.src1
+#### A masked pattern was here ####
+ Partition
+ base file name: hr=11
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ numFiles 1
+ numRows 0
+ partition_columns ds/hr
+ rawDataSize 0
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ numFiles 4
+ numPartitions 4
+ numRows 0
+ partition_columns ds/hr
+ rawDataSize 0
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 23248
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
Truncated Path -> Alias:
/src [y]
Modified: hive/branches/tez/ql/src/test/results/clientpositive/join33.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/join33.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/join33.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/join33.q.out Mon Aug 5 22:31:28 2013
@@ -24,8 +24,8 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-7 is a root stage
- Stage-6 depends on stages: Stage-7
- Stage-0 depends on stages: Stage-6
+ Stage-5 depends on stages: Stage-7
+ Stage-0 depends on stages: Stage-5
Stage-2 depends on stages: Stage-0
STAGE PLANS:
@@ -66,7 +66,7 @@ STAGE PLANS:
1 [Column[value]]
Position of Big Table: 0
- Stage: Stage-6
+ Stage: Stage-5
Map Reduce
Alias -> Map Operator Tree:
y
@@ -219,6 +219,52 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src1
name: default.src1
+#### A masked pattern was here ####
+ Partition
+ base file name: hr=11
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ numFiles 1
+ numRows 0
+ partition_columns ds/hr
+ rawDataSize 0
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.srcpart
+ numFiles 4
+ numPartitions 4
+ numRows 0
+ partition_columns ds/hr
+ rawDataSize 0
+ serialization.ddl struct srcpart { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 23248
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.srcpart
+ name: default.srcpart
Truncated Path -> Alias:
/src [y]
Modified: hive/branches/tez/ql/src/test/results/clientpositive/join_star.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/join_star.q.out?rev=1510788&r1=1510787&r2=1510788&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/join_star.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/join_star.q.out Mon Aug 5 22:31:28 2013
@@ -189,7 +189,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
+ Stage-4 depends on stages: Stage-6
Stage-0 is a root stage
STAGE PLANS:
@@ -228,7 +228,7 @@ STAGE PLANS:
1 [Column[f3]]
Position of Big Table: 0
- Stage: Stage-5
+ Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
fact
@@ -314,7 +314,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
+ Stage-4 depends on stages: Stage-6
Stage-0 is a root stage
STAGE PLANS:
@@ -353,7 +353,7 @@ STAGE PLANS:
1 [Column[f3]]
Position of Big Table: 0
- Stage: Stage-5
+ Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
fact
@@ -439,7 +439,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-6 is a root stage
- Stage-5 depends on stages: Stage-6
+ Stage-4 depends on stages: Stage-6
Stage-0 is a root stage
STAGE PLANS:
@@ -478,7 +478,7 @@ STAGE PLANS:
1 [Column[f3]]
Position of Big Table: 0
- Stage: Stage-5
+ Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
fact
@@ -580,7 +580,7 @@ ABSTRACT SYNTAX TREE:
STAGE DEPENDENCIES:
Stage-18 is a root stage
- Stage-17 depends on stages: Stage-18
+ Stage-12 depends on stages: Stage-18
Stage-0 is a root stage
STAGE PLANS:
@@ -698,7 +698,7 @@ STAGE PLANS:
1 [Column[f13]]
Position of Big Table: 0
- Stage: Stage-17
+ Stage: Stage-12
Map Reduce
Alias -> Map Operator Tree:
fact