You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2017/02/10 17:28:17 UTC
hive git commit: HIVE-15871 Cannot insert into target table because
column number/types are different with hive.merge.cardinality.check=false
(Eugene Koifman, reviewed by Wei Zheng)
Repository: hive
Updated Branches:
refs/heads/master c0198e529 -> 260717ce8
HIVE-15871 Cannot insert into target table because column number/types are different with hive.merge.cardinality.check=false (Eugene Koifman, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/260717ce
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/260717ce
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/260717ce
Branch: refs/heads/master
Commit: 260717ce8fff1d268f98d1a020843c0cca28b199
Parents: c0198e5
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Fri Feb 10 09:21:43 2017 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Fri Feb 10 09:21:43 2017 -0800
----------------------------------------------------------------------
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 10 +++++----
.../apache/hadoop/hive/ql/TestTxnCommands.java | 22 ++++++++++++++++++++
2 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/260717ce/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index e798328..1009e20 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -696,14 +696,14 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
if(numWhenMatchedDeleteClauses + numWhenMatchedUpdateClauses == 2 && extraPredicate == null) {
throw new SemanticException(ErrorMsg.MERGE_PREDIACTE_REQUIRED, ctx.getCmd());
}
- handleCardinalityViolation(rewrittenQueryStr, target, onClauseAsText, targetTable);
+ boolean validating = handleCardinalityViolation(rewrittenQueryStr, target, onClauseAsText, targetTable);
ReparseResult rr = parseRewrittenQuery(rewrittenQueryStr, ctx.getCmd());
Context rewrittenCtx = rr.rewrittenCtx;
ASTNode rewrittenTree = rr.rewrittenTree;
//set dest name mapping on new context
for(int insClauseIdx = 1, whenClauseIdx = 0;
- insClauseIdx < rewrittenTree.getChildCount() - 1/*skip cardinality violation clause*/;
+ insClauseIdx < rewrittenTree.getChildCount() - (validating ? 1 : 0/*skip cardinality violation clause*/);
insClauseIdx++, whenClauseIdx++) {
//we've added Insert clauses in order or WHEN items in whenClauses
ASTNode insertClause = (ASTNode) rewrittenTree.getChild(insClauseIdx);
@@ -825,14 +825,15 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
* This should not affect the runtime of the query as it's running in parallel with other
* branches of the multi-insert. It won't actually write any data to merge_tmp_table since the
* cardinality_violation() UDF throws an error whenever it's called killing the query
+ * @return true if another Insert clause was added
*/
- private void handleCardinalityViolation(StringBuilder rewrittenQueryStr, ASTNode target,
+ private boolean handleCardinalityViolation(StringBuilder rewrittenQueryStr, ASTNode target,
String onClauseAsString, Table targetTable)
throws SemanticException {
if(!conf.getBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK)) {
LOG.info("Merge statement cardinality violation check is disabled: " +
HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK.varname);
- return;
+ return false;
}
//this is a tmp table and thus Session scoped and acid requires SQL statement to be serial in a
// given session, i.e. the name can be fixed across all invocations
@@ -872,6 +873,7 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
catch(HiveException|MetaException e) {
throw new SemanticException(e.getMessage(), e);
}
+ return true;
}
/**
* @param onClauseAsString - because there is no clone() and we need to use in multiple places
http://git-wip-us.apache.org/repos/asf/hive/blob/260717ce/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index c110089..d2ade88 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -789,6 +789,28 @@ public class TestTxnCommands {
Assert.assertEquals(stringifyValues(rExpected), r);
}
@Test
+ public void testMergeUpdateDeleteNoCardCheck() throws Exception {
+ d.destroy();
+ HiveConf hc = new HiveConf(hiveConf);
+ hc.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, false);
+ d = new Driver(hc);
+ d.setMaxRows(10000);
+
+ int[][] baseValsOdd = {{2,2},{4,44},{5,5},{11,11}};
+ runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(baseValsOdd));
+ int[][] vals = {{2,1},{4,3},{5,6},{7,8}};
+ runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals));
+ String query = "merge into " + Table.ACIDTBL +
+ " as t using " + Table.NONACIDORCTBL + " s ON t.a = s.a " +
+ "WHEN MATCHED AND s.a < 3 THEN update set b = 0 " +
+ "WHEN MATCHED and t.a > 3 and t.a < 5 THEN DELETE ";
+ runStatementOnDriver(query);
+
+ List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+ int[][] rExpected = {{2,0},{5,6},{7,8}};
+ Assert.assertEquals(stringifyValues(rExpected), r);
+ }
+ @Test
public void testMergeDeleteUpdate() throws Exception {
int[][] sourceVals = {{2,2},{4,44},{5,5},{11,11}};
runStatementOnDriver("insert into " + Table.NONACIDORCTBL + " " + makeValuesClause(sourceVals));