You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/09/12 19:49:29 UTC
hive git commit: HIVE-17504: Skip ACID table for replication (Tao Li,
reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/master 0e0c98039 -> c8dccfdde
HIVE-17504: Skip ACID table for replication (Tao Li, reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c8dccfdd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c8dccfdd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c8dccfdd
Branch: refs/heads/master
Commit: c8dccfdde6dc41b474066dc5eb2b9078676b6022
Parents: 0e0c980
Author: Daniel Dai <da...@hortonworks.com>
Authored: Tue Sep 12 12:49:21 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Tue Sep 12 12:49:21 2017 -0700
----------------------------------------------------------------------
.../hive/ql/parse/TestReplicationScenarios.java | 36 ++++++++++++++++++++
.../apache/hadoop/hive/ql/parse/EximUtil.java | 3 +-
2 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c8dccfdd/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index aae5a7c..40fc3fa 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -2936,6 +2936,42 @@ public class TestReplicationScenarios {
verifyRun("SELECT max(a) from " + dbName + "_dupe.ptned2 where b=1", new String[]{"8"}, driverMirror);
}
+ @Test
+ public void testSkipTables() throws IOException {
+ String testName = "skipTables";
+ String dbName = createDB(testName, driver);
+
+ // Create table
+ run("CREATE TABLE " + dbName + ".acid_table (key int, value int) PARTITIONED BY (load_date date) " +
+ "CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver);
+ verifyIfTableExist(dbName, "acid_table", metaStoreClient);
+
+ // Bootstrap test
+ advanceDumpDir();
+ run("REPL DUMP " + dbName, driver);
+ String replDumpLocn = getResult(0, 0,driver);
+ String replDumpId = getResult(0, 1, true, driver);
+ LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
+ verifyIfTableNotExist(dbName + "_dupe", "acid_table", metaStoreClientMirror);
+
+ // // Create another table for incremental repl verification
+ run("CREATE TABLE " + dbName + ".acid_table_incremental (key int, value int) PARTITIONED BY (load_date date) " +
+ "CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver);
+ verifyIfTableExist(dbName, "acid_table_incremental", metaStoreClient);
+
+ // Perform REPL-DUMP/LOAD
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String incrementalDumpLocn = getResult(0,0,driver);
+ String incrementalDumpId = getResult(0,1,true,driver);
+ LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror);
+ verifyIfTableNotExist(dbName + "_dupe", "acid_table_incremental", metaStoreClientMirror);
+ }
+
private static String createDB(String name, Driver myDriver) {
LOG.info("Testing " + name);
String dbName = name + "_" + tid;
http://git-wip-us.apache.org/repos/asf/hive/blob/c8dccfdd/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index ece5495..95d394e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@ -420,7 +420,8 @@ public class EximUtil {
}
if (replicationSpec.isInReplicationScope()) {
- return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative());
+ return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative() ||
+ (tableHandle.getParameters() != null && StringUtils.equals(tableHandle.getParameters().get("transactional"), "true")));
}
if (tableHandle.isNonNative()) {