You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/04/24 08:14:03 UTC
svn commit: r768158 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/test/queries/clientnegative/ ql/src/test/queries/clientpositive/
ql/src/test/results/clientnegative/ ...
Author: zshao
Date: Fri Apr 24 06:14:03 2009
New Revision: 768158
URL: http://svn.apache.org/viewvc?rev=768158&view=rev
Log:
HIVE-376. In strict mode do not allow join without "ON" condition. (Namit Jain via zshao)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q
hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q
hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Apr 24 06:14:03 2009
@@ -18,6 +18,9 @@
HIVE-250. Shared memory java dbm for map-side joins.
(Joydeep Sen Sarma via zshao)
+ HIVE-376. In strict mode do not allow join without "ON" condition.
+ (Namit Jain via zshao)
+
IMPROVEMENTS
HIVE-389. Option to build without ivy (jssarma)
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Apr 24 06:14:03 2009
@@ -104,7 +104,7 @@
// for hive script operator
HIVETABLENAME("hive.table.name", ""),
HIVEPARTITIONNAME("hive.partition.name", ""),
- HIVEPARTITIONPRUNER("hive.partition.pruning", "nonstrict"),
+ HIVEMAPREDMODE("hive.mapred.mode", "nonstrict"),
HIVEALIAS("hive.alias", ""),
HIVEMAPSIDEAGGREGATE("hive.map.aggr", "true"),
HIVEGROUPBYSKEW("hive.groupby.skewindata", "false"),
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Fri Apr 24 06:14:03 2009
@@ -101,6 +101,12 @@
</property>
<property>
+ <name>hive.mapred.mode</name>
+ <value>nonstrict</value>
+ <description>The mode in which the hive operations are being performed. In strict mode, some risky queries are not allowed to run</description>
+</property>
+
+<property>
<name>hive.exec.script.maxerrsize</name>
<value>100000</value>
<description>Maximum number of bytes a script is allowed to emit to standard error (per map-reduce task). This prevents runaway scripts from filling logs partitions to capacity </description>
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Fri Apr 24 06:14:03 2009
@@ -72,6 +72,7 @@
ORDERBY_SORTBY_CONFLICT("Cannot have both Order By and Sort By Clauses"),
CLUSTERBY_ORDERBY_CONFLICT("Cannot have both Cluster By and Order By Clauses"),
NO_LIMIT_WITH_ORDERBY("In strict mode, limit must be specified if ORDER BY is present"),
+ NO_CARTESIAN_PRODUCT("In strict mode, cartesian product is not allowed. If you really want to perform the operation, set hive.mapred.mode=nonstrict"),
UNION_NOTIN_SUBQ("Top level Union is not supported currently; use a subquery for the union"),
INVALID_INPUT_FORMAT_TYPE("Input Format must implement InputFormat"),
INVALID_OUTPUT_FORMAT_TYPE("Output Format must implement HiveOutputFormat, otherwise it should be either IgnoreKeyTextOutputFormat or SequenceFileOutputFormat"),
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Apr 24 06:14:03 2009
@@ -625,7 +625,7 @@
ASTNode whexp = (ASTNode)qbp.getWhrForClause(clause);
if (pruner.getTable().isPartitioned() &&
- conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict") &&
+ conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict") &&
(whexp == null || !pruner.hasPartitionPredicate((ASTNode)whexp.getChild(0)))) {
throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE.getMsg(whexp != null ? whexp : qbp.getSelForClause(clause),
" for Alias " + alias + " Table " + pruner.getTable().getName()));
@@ -2421,7 +2421,7 @@
assert numReducers == 1;
// in strict mode, in the presence of order by, limit must be specified
Integer limit = qb.getParseInfo().getDestLimit(dest);
- if (conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict") && limit == null)
+ if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict") && limit == null)
throw new SemanticException(ErrorMsg.NO_LIMIT_WITH_ORDERBY.getMsg(sortExprs));
}
}
@@ -2588,9 +2588,20 @@
}
}
+ int numReds = -1;
+
+ // Use only 1 reducer in case of cartesian product
+ if (reduceKeys.size() == 0) {
+ numReds = 1;
+
+ // Cartesian product is not supported in strict mode
+ if (conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict"))
+ throw new SemanticException(ErrorMsg.NO_CARTESIAN_PRODUCT.getMsg());
+ }
+
ReduceSinkOperator rsOp = (ReduceSinkOperator)putOpInsertMap(
OperatorFactory.getAndMakeChild(
- PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(), -1),
+ PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(), numReds),
new RowSchema(outputRS.getColumnInfos()),
child), outputRS);
rsOp.setColumnExprMap(colExprMap);
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_join.q Fri Apr 24 06:14:03 2009
@@ -0,0 +1,3 @@
+set hive.mapred.mode=strict;
+
+SELECT * FROM src src1 JOIN src src2;
Modified: hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_orderby.q Fri Apr 24 06:14:03 2009
@@ -1,4 +1,4 @@
-set hive.partition.pruning=strict;
+set hive.mapred.mode=strict;
EXPLAIN
SELECT src.key, src.value from src order by src.key;
Modified: hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q?rev=768158&r1=768157&r2=768158&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/strict_pruning.q Fri Apr 24 06:14:03 2009
@@ -1,4 +1,4 @@
-set hive.partition.pruning=strict;
+set hive.mapred.mode=strict;
EXPLAIN
SELECT count(1) FROM srcPART;
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join23.q Fri Apr 24 06:14:03 2009
@@ -0,0 +1,4 @@
+EXPLAIN
+SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10;
+
+SELECT * FROM src src1 JOIN src src2 WHERE src1.key < 10 and src2.key < 10;
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/strict_join.q.out Fri Apr 24 06:14:03 2009
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: In strict mode, cartesian product is not allowed. If you really want to perform the operation, set hive.mapred.mode=nonstrict
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out?rev=768158&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join23.q.out Fri Apr 24 06:14:03 2009
@@ -0,0 +1,162 @@
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src src1) (TOK_TABREF src src2))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (and (< (. (TOK_TABLE_OR_COL src1) key) 10) (< (. (TOK_TABLE_OR_COL src2) key) 10)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src2
+ Reduce Output Operator
+ sort order:
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ src1
+ Reduce Output Operator
+ sort order:
+ tag: 0
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE.0} {VALUE.1}
+ 1 {VALUE.0} {VALUE.1}
+ Filter Operator
+ predicate:
+ expr: ((UDFToDouble(0) < UDFToDouble(10)) and (UDFToDouble(2) < UDFToDouble(10)))
+ type: boolean
+ Select Operator
+ expressions:
+ expr: 0
+ type: string
+ expr: 1
+ type: string
+ expr: 2
+ type: string
+ expr: 3
+ type: string
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+5 val_5 2 val_2
+5 val_5 5 val_5
+5 val_5 9 val_9
+5 val_5 4 val_4
+5 val_5 8 val_8
+5 val_5 0 val_0
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 0 val_0
+5 val_5 0 val_0
+0 val_0 2 val_2
+0 val_0 5 val_5
+0 val_0 9 val_9
+0 val_0 4 val_4
+0 val_0 8 val_8
+0 val_0 0 val_0
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 0 val_0
+0 val_0 0 val_0
+8 val_8 2 val_2
+8 val_8 5 val_5
+8 val_8 9 val_9
+8 val_8 4 val_4
+8 val_8 8 val_8
+8 val_8 0 val_0
+8 val_8 5 val_5
+8 val_8 5 val_5
+8 val_8 0 val_0
+8 val_8 0 val_0
+0 val_0 2 val_2
+0 val_0 5 val_5
+0 val_0 9 val_9
+0 val_0 4 val_4
+0 val_0 8 val_8
+0 val_0 0 val_0
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 0 val_0
+0 val_0 0 val_0
+2 val_2 2 val_2
+2 val_2 5 val_5
+2 val_2 9 val_9
+2 val_2 4 val_4
+2 val_2 8 val_8
+2 val_2 0 val_0
+2 val_2 5 val_5
+2 val_2 5 val_5
+2 val_2 0 val_0
+2 val_2 0 val_0
+5 val_5 2 val_2
+5 val_5 5 val_5
+5 val_5 9 val_9
+5 val_5 4 val_4
+5 val_5 8 val_8
+5 val_5 0 val_0
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 0 val_0
+5 val_5 0 val_0
+5 val_5 2 val_2
+5 val_5 5 val_5
+5 val_5 9 val_9
+5 val_5 4 val_4
+5 val_5 8 val_8
+5 val_5 0 val_0
+5 val_5 5 val_5
+5 val_5 5 val_5
+5 val_5 0 val_0
+5 val_5 0 val_0
+0 val_0 2 val_2
+0 val_0 5 val_5
+0 val_0 9 val_9
+0 val_0 4 val_4
+0 val_0 8 val_8
+0 val_0 0 val_0
+0 val_0 5 val_5
+0 val_0 5 val_5
+0 val_0 0 val_0
+0 val_0 0 val_0
+9 val_9 2 val_2
+9 val_9 5 val_5
+9 val_9 9 val_9
+9 val_9 4 val_4
+9 val_9 8 val_8
+9 val_9 0 val_0
+9 val_9 5 val_5
+9 val_9 5 val_5
+9 val_9 0 val_0
+9 val_9 0 val_0
+4 val_4 2 val_2
+4 val_4 5 val_5
+4 val_4 9 val_9
+4 val_4 4 val_4
+4 val_4 8 val_8
+4 val_4 0 val_0
+4 val_4 5 val_5
+4 val_4 5 val_5
+4 val_4 0 val_0
+4 val_4 0 val_0