You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2009/12/22 02:30:33 UTC
svn commit: r893054 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/test/queries/clientnegative/ ql/src/test/results/clientnegative/
Author: heyongqiang
Date: Tue Dec 22 01:30:32 2009
New Revision: 893054
URL: http://svn.apache.org/viewvc?rev=893054&view=rev
Log:
Hive-913 no error if user specifies same output table mutliple times
Added:
hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert1.q
hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert2.q
hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert3.q
hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert1.q.out
hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert2.q.out
hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert3.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=893054&r1=893053&r2=893054&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Dec 22 01:30:32 2009
@@ -342,6 +342,9 @@
HIVE-1001 CombinedHiveInputFormat should parse the inputpath correctly
(namit via He Yongqiang)
+ HIVE-913 no error if user specifies same output table mutliple times
+ (namit via He Yongqiang)
+
Release 0.4.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=893054&r1=893053&r2=893054&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Tue Dec 22 01:30:32 2009
@@ -119,6 +119,7 @@
UDTF_ALIAS_MISMATCH("The number of aliases supplied in the AS clause does not match the number of columns output by the UDTF"),
LATERAL_VIEW_WITH_JOIN("Join with a lateral view is not supported"),
LATERAL_VIEW_INVALID_CHILD("Lateral view AST with invalid child"),
+ OUTPUT_SPECIFIED_MULTIPLE_TIMES("The same output cannot be present multiple times: "),
INVALID_AS("AS clause has an invalid number of aliases");
private String mesg;
private String SQLState;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=893054&r1=893053&r2=893054&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Dec 22 01:30:32 2009
@@ -332,7 +332,7 @@
* Goes though the tabref tree and finds the alias for the table. Once found,
* it records the table name-> alias association in aliasToTabs. It also makes
* an association from the alias to the table AST in parse info.
- *
+ *
* @return the alias of the table
*/
private String processTable(QB qb, ASTNode tabref) throws SemanticException {
@@ -388,7 +388,7 @@
qb.setTabAlias(alias, table_name);
qb.getParseInfo().setSrcForAlias(alias, tableTree);
-
+
return alias;
}
@@ -412,7 +412,7 @@
}
// Insert this map into the stats
qb.setSubqAlias(alias, qbexpr);
-
+
return alias;
}
@@ -432,7 +432,7 @@
/**
* Given the AST with TOK_JOIN as the root, get all the aliases for the tables
* or subqueries in the join.
- *
+ *
* @param qb
* @param join
* @throws SemanticException
@@ -455,34 +455,34 @@
// is not supported. Instead, the lateral view must be in a subquery
// SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a
// JOIN src2 ...
- throw new
+ throw new
SemanticException(ErrorMsg.LATERAL_VIEW_WITH_JOIN.getMsg(join));
} else if (isJoinToken(child)) {
processJoin(qb, child);
}
}
}
-
+
/**
* Given the AST with TOK_LATERAL_VIEW as the root, get the alias for the
* table or subquery in the lateral view and also make a mapping from the
* alias to all the lateral view AST's
- *
+ *
* @param qb
* @param lateralView
* @return the alias for the table/subquery
* @throws SemanticException
*/
-
- private String processLateralView(QB qb, ASTNode lateralView)
+
+ private String processLateralView(QB qb, ASTNode lateralView)
throws SemanticException {
int numChildren = lateralView.getChildCount();
-
+
assert(numChildren == 2);
ASTNode next = (ASTNode) lateralView.getChild(1);
-
+
String alias = null;
-
+
switch(next.getToken().getType()) {
case HiveParser.TOK_TABREF:
alias = processTable(qb, next);
@@ -503,7 +503,7 @@
/**
* Phase 1: (including, but not limited to):
- *
+ *
* 1. Gets all the aliases for all the tables / subqueries and makes the
* appropriate mapping in aliasToTabs, aliasToSubq
* 2. Gets the location of the destination and names the clase "inclause" + i
@@ -511,9 +511,9 @@
* actual aggregation AST
* 4. Creates a mapping from the clause name to the select expression AST in
* destToSelExpr
- * 5. Creates a mapping from a table alias to the lateral view AST's in
+ * 5. Creates a mapping from a table alias to the lateral view AST's in
* aliasToLateralViews
- *
+ *
* @param ast
* @param qb
* @param ctx_1
@@ -761,7 +761,7 @@
return false;
}
-
+
@SuppressWarnings("nls")
private void parseJoinCondPopulateAlias(QBJoinTree joinTree,
ASTNode condn, Vector<String> leftAliases, Vector<String> rightAliases,
@@ -1407,16 +1407,16 @@
private Operator<?> genSelectPlan(String dest, QB qb,
Operator<?> input) throws SemanticException {
ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
-
+
Operator<?> op = genSelectPlan(selExprList, qb, input);
LOG.debug("Created Select Plan for clause: " + dest);
return op;
}
@SuppressWarnings("nls")
private Operator<?> genSelectPlan(ASTNode selExprList, QB qb,
- Operator<?> input) throws SemanticException {
+ Operator<?> input) throws SemanticException {
LOG.debug("tree: " + selExprList.toStringTree());
-
+
ArrayList<exprNodeDesc> col_list = new ArrayList<exprNodeDesc>();
RowResolver out_rwsch = new RowResolver();
ASTNode trfm = null;
@@ -1436,9 +1436,9 @@
if (isInTransform) {
trfm = (ASTNode) selExprList.getChild(posn).getChild(0);
}
-
+
// Detect queries of the form SELECT udtf(col) AS ...
- // by looking for a function as the first child, and then checking to see
+ // by looking for a function as the first child, and then checking to see
// if the function is a Generic UDTF. It's not as clean as TRANSFORM due to
// the lack of a special token.
boolean isUDTF = false;
@@ -1465,7 +1465,7 @@
}
// Require an AS for UDTFs for column aliases
ASTNode selExpr = (ASTNode) selExprList.getChild(posn);
- if (selExpr.getChildCount() < 2) {
+ if (selExpr.getChildCount() < 2) {
throw new SemanticException(ErrorMsg.UDTF_REQUIRE_AS.getMsg());
}
// Get the column / table aliases from the expression. Start from 1 as
@@ -1478,7 +1478,7 @@
break;
case HiveParser.TOK_TABALIAS:
assert(selExprChild.getChildCount() == 1);
- udtfTableAlias =
+ udtfTableAlias =
unescapeIdentifier(selExprChild.getChild(0).getText());
break;
default:
@@ -1488,7 +1488,7 @@
LOG.debug("UDTF table alias is " + udtfTableAlias);
LOG.debug("UDTF col aliases are " + udtfColAliases);
}
-
+
// The list of expressions after SELECT or SELECT TRANSFORM.
ASTNode exprList;
if (isInTransform) {
@@ -1515,7 +1515,7 @@
if (!isUDTF && child.getChildCount() > 2) {
throw new SemanticException(ErrorMsg.INVALID_AS.getMsg());
}
-
+
// The real expression
ASTNode expr;
String tabAlias;
@@ -1532,7 +1532,7 @@
// Get rid of TOK_SELEXPR
expr = (ASTNode)child.getChild(0);
}
-
+
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
pos = genColListRegex(".*",
expr.getChildCount() == 0 ? null : unescapeIdentifier(expr.getChild(0).getText().toLowerCase()),
@@ -1598,7 +1598,7 @@
}
if (isUDTF) {
- output = genUDTFPlan(genericUDTF, udtfTableAlias, udtfColAliases, qb,
+ output = genUDTFPlan(genericUDTF, udtfTableAlias, udtfColAliases, qb,
output);
}
LOG.debug("Created Select Plan row schema: " + out_rwsch.toString());
@@ -2717,7 +2717,9 @@
ctx.getExternalTmpFileURI(dest_path.toUri()),
table_desc,
new HashMap<String, String>()));
- outputs.add(new WriteEntity(dest_tab));
+ if (!outputs.add(new WriteEntity(dest_tab))) {
+ throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES.getMsg(dest_tab.getName()));
+ }
break;
}
case QBMetaData.DEST_PARTITION: {
@@ -2736,7 +2738,9 @@
(new loadTableDesc(queryTmpdir,
ctx.getExternalTmpFileURI(dest_path.toUri()),
table_desc, dest_part.getSpec()));
- outputs.add(new WriteEntity(dest_part));
+ if (!outputs.add(new WriteEntity(dest_part))) {
+ throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES.getMsg(dest_tab.getName() + "@" + dest_part.getName()));
+ }
break;
}
case QBMetaData.DEST_LOCAL_FILE:
@@ -2837,7 +2841,9 @@
table_desc = PlanUtils.getTableDesc(tblDesc, cols, colTypes);
}
- outputs.add(new WriteEntity(destStr, !isDfsDir));
+ if (!outputs.add(new WriteEntity(destStr, !isDfsDir))) {
+ throw new SemanticException(ErrorMsg.OUTPUT_SPECIFIED_MULTIPLE_TIMES.getMsg(destStr));
+ }
break;
}
default:
@@ -2979,7 +2985,7 @@
}
private Operator genUDTFPlan(GenericUDTF genericUDTF, String outputTableAlias,
- ArrayList<String> colAliases, QB qb, Operator input)
+ ArrayList<String> colAliases, QB qb, Operator input)
throws SemanticException {
// No GROUP BY / DISTRIBUTE BY / SORT BY / CLUSTER BY
@@ -2999,11 +3005,11 @@
if (!qbp.getAliasToLateralViews().isEmpty()) {
throw new SemanticException(ErrorMsg.UDTF_LATERAL_VIEW.getMsg());
}
-
+
LOG.debug("Table alias: " + outputTableAlias + " Col aliases: " + colAliases);
-
- // Use the RowResolver from the input operator to generate a input
- // ObjectInspector that can be used to initialize the UDTF. Then, the
+
+ // Use the RowResolver from the input operator to generate a input
+ // ObjectInspector that can be used to initialize the UDTF. Then, the
// resulting output object inspector can be used to make the RowResolver
// for the UDTF operator
@@ -3019,7 +3025,7 @@
inputCols.get(i).getType());
}
StructObjectInspector outputOI = genericUDTF.initialize(colOIs);
-
+
// Make sure that the number of column aliases in the AS clause matches
// the number of columns output by the UDTF
int numUdtfCols = outputOI.getAllStructFieldRefs().size();
@@ -3029,26 +3035,26 @@
"expected " + numUdtfCols + " aliases " +
"but got " + numSuppliedAliases));
}
-
+
// Generate the output column info's / row resolver using internal names.
ArrayList<ColumnInfo> udtfCols = new ArrayList<ColumnInfo>();
Iterator<String> colAliasesIter = colAliases.iterator();
for (StructField sf : outputOI.getAllStructFieldRefs()) {
-
+
String colAlias = colAliasesIter.next();
assert(colAlias != null);
-
+
// Since the UDTF operator feeds into a LVJ operator that will rename
// all the internal names, we can just use field name from the UDTF's OI
// as the internal name
- ColumnInfo col = new ColumnInfo(sf.getFieldName(),
+ ColumnInfo col = new ColumnInfo(sf.getFieldName(),
TypeInfoUtils.getTypeInfoFromObjectInspector(
sf.getFieldObjectInspector()),
outputTableAlias, false);
udtfCols.add(col);
}
-
+
// Create the row resolver for this operator from the output columns
RowResolver out_rwsch = new RowResolver();
for (int i=0; i<udtfCols.size(); i++) {
@@ -3058,7 +3064,7 @@
// Add the UDTFOperator to the operator DAG
Operator<?> udtf =
putOpInsertMap(OperatorFactory.getAndMakeChild(
- new udtfDesc(genericUDTF),
+ new udtfDesc(genericUDTF),
new RowSchema(out_rwsch.getColumnInfos()),
input), out_rwsch);
return udtf;
@@ -3787,7 +3793,7 @@
filters.add(new Vector<ASTNode>());
joinTree.setFilters(filters);
- ASTNode joinCond = (ASTNode) joinParseTree.getChild(2);
+ ASTNode joinCond = (ASTNode) joinParseTree.getChild(2);
Vector<String> leftSrc = new Vector<String>();
parseJoinCondition(joinTree, joinCond, leftSrc);
if (leftSrc.size() == 1)
@@ -4523,7 +4529,7 @@
if (tabBucketCols.size() == 0 && sampleExprs.size() == 0) {
throw new SemanticException(ErrorMsg.NON_BUCKETED_TABLE.getMsg() + " " + tab.getName());
}
-
+
if (num > den) {
throw new SemanticException(ErrorMsg.BUCKETED_NUMBERATOR_BIGGER_DENOMINATOR.getMsg() + " " + tab.getName());
}
@@ -4656,7 +4662,7 @@
for (String alias : qb.getTabAliases()) {
aliasToOpInfo.put(alias, genTablePlan(alias, qb));
}
-
+
// For all the source tables that have a lateral view, attach the
// appropriate operators to the TS
genLateralViewPlans(aliasToOpInfo, qb);
@@ -4695,14 +4701,14 @@
/**
* Generates the operator DAG needed to implement lateral views and attaches
* it to the TS operator.
- *
+ *
* @param aliasToOpInfo A mapping from a table alias to the TS operator. This
* function replaces the operator mapping as necessary
* @param qb
* @throws SemanticException
*/
-
- void genLateralViewPlans(HashMap<String, Operator> aliasToOpInfo, QB qb)
+
+ void genLateralViewPlans(HashMap<String, Operator> aliasToOpInfo, QB qb)
throws SemanticException {
Map<String, ArrayList<ASTNode>> aliasToLateralViews =
qb.getParseInfo().getAliasToLateralViews();
@@ -4713,7 +4719,7 @@
ArrayList<ASTNode> lateralViews = aliasToLateralViews.get(alias);
if (lateralViews != null) {
Operator op = e.getValue();
-
+
for (ASTNode lateralViewTree : aliasToLateralViews.get(alias)) {
// There are 2 paths from the TS operator (or a previous LVJ operator)
// to the same LateralViewJoinOperator.
@@ -4731,49 +4737,49 @@
putOpInsertMap(OperatorFactory.getAndMakeChild(
new selectDesc(true),
new RowSchema(allPathRR.getColumnInfos()),
- op), allPathRR);
+ op), allPathRR);
// Get the UDTF Path
QB blankQb = new QB(null, null, false);
- Operator udtfPath =
+ Operator udtfPath =
genSelectPlan((ASTNode)lateralViewTree.getChild(0), blankQb, op);
RowResolver udtfPathRR = opParseCtx.get(udtfPath).getRR();
- // Merge the two into the lateral view join
+ // Merge the two into the lateral view join
// The cols of the merged result will be the combination of both the
// cols of the UDTF path and the cols of the all path. The internal
// names have to be changed to avoid conflicts
RowResolver lateralViewRR = new RowResolver();
ArrayList<String> outputInternalColNames = new ArrayList<String>();
-
- LVmergeRowResolvers(allPathRR, lateralViewRR,
+
+ LVmergeRowResolvers(allPathRR, lateralViewRR,
outputInternalColNames);
- LVmergeRowResolvers(udtfPathRR, lateralViewRR,
+ LVmergeRowResolvers(udtfPathRR, lateralViewRR,
outputInternalColNames);
Operator lateralViewJoin =
putOpInsertMap(OperatorFactory.getAndMakeChild(
new lateralViewJoinDesc(outputInternalColNames),
new RowSchema(lateralViewRR.getColumnInfos()),
- allPath, udtfPath), lateralViewRR);
+ allPath, udtfPath), lateralViewRR);
op = lateralViewJoin;
}
e.setValue(op);
}
}
}
-
+
/**
* A helper function that gets all the columns and respective aliases in the
- * source and puts them into dest. It renames the internal names of the
+ * source and puts them into dest. It renames the internal names of the
* columns based on getColumnInternalName(position).
- *
+ *
* Note that this helper method relies on RowResolver.getColumnInfos()
- * returning the columns in the same order as they will be passed in the
+ * returning the columns in the same order as they will be passed in the
* operator DAG.
- *
+ *
* @param source
* @param dest
* @param outputColNames - a list to which the new internal column names will
@@ -4781,10 +4787,10 @@
* resolver
*/
private void LVmergeRowResolvers(RowResolver source, RowResolver dest,
- ArrayList<String> outputInternalColNames) {
+ ArrayList<String> outputInternalColNames) {
Vector<ColumnInfo> cols = source.getColumnInfos();
for (ColumnInfo c : cols) {
- String internalName =
+ String internalName =
getColumnInternalName(outputInternalColNames.size());
outputInternalColNames.add(internalName);
ColumnInfo newCol = new ColumnInfo(internalName, c.getType(),
@@ -4795,7 +4801,7 @@
dest.put(tableAlias, colAlias, newCol);
}
}
-
+
private Operator<? extends Serializable> getReduceSink(Operator<? extends Serializable> top) {
if (top.getClass() == ReduceSinkOperator.class) {
// Get the operator following the reduce sink
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert1.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert1.q?rev=893054&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert1.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert1.q Tue Dec 22 01:30:32 2009
@@ -0,0 +1,7 @@
+
+create table dest1_din1(key int, value string);
+
+from src
+insert overwrite table dest1_din1 select key, value
+insert overwrite table dest1_din1 select key, value;
+
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert2.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert2.q?rev=893054&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert2.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert2.q Tue Dec 22 01:30:32 2009
@@ -0,0 +1,6 @@
+
+create table dest1_din2(key int, value string) partitioned by (ds string);
+
+from src
+insert overwrite table dest1_din2 partition (ds='1') select key, value
+insert overwrite table dest1_din2 partition (ds='1') select key, value;
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert3.q?rev=893054&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert3.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/duplicate_insert3.q Tue Dec 22 01:30:32 2009
@@ -0,0 +1,4 @@
+
+from src
+insert overwrite directory '/tmp/dest1' select key, value
+insert overwrite directory '/tmp/dest1' select key, value;
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert1.q.out?rev=893054&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert1.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert1.q.out Tue Dec 22 01:30:32 2009
@@ -0,0 +1,6 @@
+PREHOOK: query: create table dest1_din1(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table dest1_din1(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1_din1
+FAILED: Error in semantic analysis: The same output cannot be present multiple times: dest1_din1
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert2.q.out?rev=893054&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert2.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert2.q.out Tue Dec 22 01:30:32 2009
@@ -0,0 +1,6 @@
+PREHOOK: query: create table dest1_din2(key int, value string) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table dest1_din2(key int, value string) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1_din2
+FAILED: Error in semantic analysis: The same output cannot be present multiple times: dest1_din2@ds=1
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert3.q.out?rev=893054&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert3.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/duplicate_insert3.q.out Tue Dec 22 01:30:32 2009
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: The same output cannot be present multiple times: /tmp/dest1