You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/11/09 00:30:41 UTC
svn commit: r1540218 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/parse/ASTNode.java
test/queries/clientpositive/subq_where_serialization.q
test/results/clientpositive/subq_where_serialization.q.out
Author: hashutosh
Date: Fri Nov 8 23:30:40 2013
New Revision: 1540218
URL: http://svn.apache.org/r1540218
Log:
HIVE-5779 : Subquery in where clause with distinct fails with mapjoin turned on with serialization error. (Ashutosh Chauhan via Harish Butani)
Added:
hive/trunk/ql/src/test/queries/clientpositive/subq_where_serialization.q
hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java?rev=1540218&r1=1540217&r2=1540218&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java Fri Nov 8 23:30:40 2013
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
public class ASTNode extends CommonTree implements Node,Serializable {
private static final long serialVersionUID = 1L;
- private ASTNodeOrigin origin;
+ private transient ASTNodeOrigin origin;
public ASTNode() {
}
Added: hive/trunk/ql/src/test/queries/clientpositive/subq_where_serialization.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/subq_where_serialization.q?rev=1540218&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/subq_where_serialization.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/subq_where_serialization.q Fri Nov 8 23:30:40 2013
@@ -0,0 +1,5 @@
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask.size=10000000;
+explain select src.key from src where src.key in ( select distinct key from src);
+
+set hive.auto.convert.join=false;
Added: hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out?rev=1540218&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out Fri Nov 8 23:30:40 2013
@@ -0,0 +1,190 @@
+PREHOOK: query: explain select src.key from src where src.key in ( select distinct key from src)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select src.key from src where src.key in ( select distinct key from src)
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src) key))) (TOK_WHERE (TOK_SUBQUERY_EXPR (TOK_SUBQUERY_OP in) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECTDI (TOK_SELEXPR (TOK_TABLE_OR_COL key))))) (. (TOK_TABLE_OR_COL src) key)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-5 depends on stages: Stage-1 , consists of Stage-6, Stage-2
+ Stage-6 has a backup stage: Stage-2
+ Stage-4 depends on stages: Stage-6
+ Stage-2
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ sq_1:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ mode: hash
+ outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: -1
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Conditional Operator
+
+ Stage: Stage-6
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col0]]
+ Position of Big Table: 0
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Map Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[_col0]]
+ outputColumnNames: _col0
+ Position of Big Table: 0
+ Filter Operator
+ predicate:
+ expr: (1 = 1)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ $INTNAME
+ TableScan
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ tag: 1
+ src
+ TableScan
+ alias: src
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0}
+ 1
+ handleSkewJoin: false
+ outputColumnNames: _col0
+ Filter Operator
+ predicate:
+ expr: (1 = 1)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+