You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2017/04/27 21:42:30 UTC
hive git commit: HIVE-16542 make merge that targets acid 2.0 table
fail-fast (Eugene Koifman, reviewed by Wei Zheng)
Repository: hive
Updated Branches:
refs/heads/master 2f5a2a79d -> fefeb2ac1
HIVE-16542 make merge that targets acid 2.0 table fail-fast (Eugene Koifman, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fefeb2ac
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fefeb2ac
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fefeb2ac
Branch: refs/heads/master
Commit: fefeb2ac1b678b139b553a4dcd16880a4a73ea7b
Parents: 2f5a2a7
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu Apr 27 14:42:25 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu Apr 27 14:42:25 2017 -0700
----------------------------------------------------------------------
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 14 +++++++
.../apache/hadoop/hive/ql/TestTxnCommands.java | 1 -
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 3 +-
.../ql/TestTxnCommands2WithSplitUpdate.java | 41 ++++++++++++++++++++
4 files changed, 56 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fefeb2ac/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index 64f1bdd..0541a40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -32,15 +32,18 @@ import org.antlr.runtime.TokenRewriteStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.hooks.Entity;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
@@ -662,9 +665,11 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
*/
String extraPredicate = null;
int numWhenMatchedUpdateClauses = 0, numWhenMatchedDeleteClauses = 0;
+ int numInsertClauses = 0;
for(ASTNode whenClause : whenClauses) {
switch (getWhenClauseOperation(whenClause).getType()) {
case HiveParser.TOK_INSERT:
+ numInsertClauses++;
handleInsert(whenClause, rewrittenQueryStr, target, onClause, targetTable, targetName, onClauseAsText);
break;
case HiveParser.TOK_UPDATE:
@@ -691,6 +696,15 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
if(numWhenMatchedUpdateClauses > 1) {
throw new SemanticException(ErrorMsg.MERGE_TOO_MANY_UPDATE, ctx.getCmd());
}
+ assert numInsertClauses < 2;
+ if(numInsertClauses == 1 && numWhenMatchedUpdateClauses == 1) {
+ if(AcidUtils.getAcidOperationalProperties(targetTable).isSplitUpdate()) {
+ throw new IllegalStateException("Tables with " +
+ hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "=" +
+ TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY + " currently do not " +
+ "support MERGE with both Insert and Update clauses.");
+ }
+ }
}
if(numWhenMatchedDeleteClauses + numWhenMatchedUpdateClauses == 2 && extraPredicate == null) {
throw new SemanticException(ErrorMsg.MERGE_PREDIACTE_REQUIRED, ctx.getCmd());
http://git-wip-us.apache.org/repos/asf/hive/blob/fefeb2ac/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 6dae7ba..c8bc119 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -103,7 +103,6 @@ public class TestTxnCommands {
hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
hiveConf
http://git-wip-us.apache.org/repos/asf/hive/blob/fefeb2ac/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 6718ae9..e2db5b7 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -113,7 +113,6 @@ public class TestTxnCommands2 {
hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
hiveConf.setVar(HiveConf.ConfVars.HIVEINPUTFORMAT, HiveInputFormat.class.getName());
@@ -1676,7 +1675,7 @@ public class TestTxnCommands2 {
return 0;
}
}
- private String makeValuesClause(int[][] rows) {
+ String makeValuesClause(int[][] rows) {
assert rows.length > 0;
StringBuilder sb = new StringBuilder("values");
for(int[] row : rows) {
http://git-wip-us.apache.org/repos/asf/hive/blob/fefeb2ac/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
index 67e917c..e516f18 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
@@ -540,4 +540,45 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 {
resultCount = 2;
Assert.assertEquals(resultCount, Integer.parseInt(rs.get(0)));
}
+ @Ignore("HIVE-14947")
+ @Test
+ @Override
+ public void testDynamicPartitionsMerge() throws Exception {}
+ @Ignore("HIVE-14947")
+ @Test
+ @Override
+ public void testDynamicPartitionsMerge2() throws Exception {}
+ @Ignore("HIVE-14947")
+ @Test
+ @Override
+ public void testMerge() throws Exception {}
+
+ /**
+ * todo: remove this test once HIVE-14947 is done (parent class has a better version)
+ */
+ @Test
+ @Override
+ public void testMerge2() throws Exception {
+ int[][] baseValsOdd = {{5,5},{11,11}};
+ int[][] baseValsEven = {{2,2},{4,44}};
+ runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='odd') " + makeValuesClause(baseValsOdd));
+ runStatementOnDriver("insert into " + Table.NONACIDPART2 + " PARTITION(p2='even') " + makeValuesClause(baseValsEven));
+ int[][] vals = {{2,1},{4,3},{5,6},{7,8}};
+ runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(vals));
+ List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+ Assert.assertEquals(stringifyValues(vals), r);
+ String query = "merge into " + Table.ACIDTBL +
+ " using " + Table.NONACIDPART2 + " source ON " + Table.ACIDTBL + ".a = source.a2 " +
+ "WHEN MATCHED THEN UPDATE set b = source.b2 ";
+ r = runStatementOnDriver(query);
+
+ r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+ int[][] rExpected = {{2,2},{4,44},{5,5},{7,8}};
+ Assert.assertEquals(stringifyValues(rExpected), r);
+
+ }
+ @Ignore("HIVE-14947")
+ @Test
+ @Override
+ public void testMergeWithPredicate() throws Exception {}
}