You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/09/18 22:45:08 UTC
hive git commit: HIVE-17422: Skip non-native/temporary tables for all
major table/partition related scenarios (Tao Li, reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/master 20b84523d -> 5971015ee
HIVE-17422: Skip non-native/temporary tables for all major table/partition related scenarios (Tao Li, reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5971015e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5971015e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5971015e
Branch: refs/heads/master
Commit: 5971015ee50dbbf729625a94dfc891d994e145a9
Parents: 20b8452
Author: Daniel Dai <da...@hortonworks.com>
Authored: Mon Sep 18 15:44:54 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Mon Sep 18 15:44:54 2017 -0700
----------------------------------------------------------------------
.../hive/ql/parse/TestReplicationScenarios.java | 21 ++++++++--
.../repl/dump/events/AddPartitionHandler.java | 18 ++++++---
.../repl/dump/events/AlterPartitionHandler.java | 6 ++-
.../repl/dump/events/AlterTableHandler.java | 40 +++++++++++---------
.../parse/repl/dump/events/InsertHandler.java | 5 +++
5 files changed, 63 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5971015e/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index ba2b3f3..4b687d6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -3019,7 +3019,22 @@ public class TestReplicationScenarios {
run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'", driverMirror);
verifyIfTableNotExist(dbName + "_dupe", "acid_table", metaStoreClientMirror);
- // // Create another table for incremental repl verification
+ // Test alter table
+ run("ALTER TABLE " + dbName + ".acid_table RENAME TO " + dbName + ".acid_table_rename", driver);
+ verifyIfTableExist(dbName, "acid_table_rename", metaStoreClient);
+
+ // Perform REPL-DUMP/LOAD
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
+ String incrementalDumpLocn = getResult(0,0,driver);
+ String incrementalDumpId = getResult(0,1,true,driver);
+ LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId);
+ run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
+ printOutput(driverMirror);
+ run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror);
+ verifyIfTableNotExist(dbName + "_dupe", "acid_table_rename", metaStoreClientMirror);
+
+ // Create another table for incremental repl verification
run("CREATE TABLE " + dbName + ".acid_table_incremental (key int, value int) PARTITIONED BY (load_date date) " +
"CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver);
verifyIfTableExist(dbName, "acid_table_incremental", metaStoreClient);
@@ -3027,8 +3042,8 @@ public class TestReplicationScenarios {
// Perform REPL-DUMP/LOAD
advanceDumpDir();
run("REPL DUMP " + dbName + " FROM " + replDumpId, driver);
- String incrementalDumpLocn = getResult(0,0,driver);
- String incrementalDumpId = getResult(0,1,true,driver);
+ incrementalDumpLocn = getResult(0,0,driver);
+ incrementalDumpId = getResult(0,1,true,driver);
LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId);
run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror);
printOutput(driverMirror);
http://git-wip-us.apache.org/repos/asf/hive/blob/5971015e/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
index 72368af..def8384 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java
@@ -44,13 +44,9 @@ class AddPartitionHandler extends AbstractEventHandler {
@Override
public void handle(Context withinContext) throws Exception {
- AddPartitionMessage apm = deserializer.getAddPartitionMessage(event.getMessage());
LOG.info("Processing#{} ADD_PARTITION message : {}", fromEventId(), event.getMessage());
- Iterable<org.apache.hadoop.hive.metastore.api.Partition> ptns = apm.getPartitionObjs();
- if ((ptns == null) || (!ptns.iterator().hasNext())) {
- LOG.debug("Event#{} was an ADD_PTN_EVENT with no partitions");
- return;
- }
+
+ AddPartitionMessage apm = deserializer.getAddPartitionMessage(event.getMessage());
org.apache.hadoop.hive.metastore.api.Table tobj = apm.getTableObj();
if (tobj == null) {
LOG.debug("Event#{} was a ADD_PTN_EVENT with no table listed");
@@ -58,6 +54,16 @@ class AddPartitionHandler extends AbstractEventHandler {
}
final Table qlMdTable = new Table(tobj);
+ if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) {
+ return;
+ }
+
+ Iterable<org.apache.hadoop.hive.metastore.api.Partition> ptns = apm.getPartitionObjs();
+ if ((ptns == null) || (!ptns.iterator().hasNext())) {
+ LOG.debug("Event#{} was an ADD_PTN_EVENT with no partitions");
+ return;
+ }
+
Iterable<Partition> qlPtns = Iterables.transform(
ptns,
new Function<org.apache.hadoop.hive.metastore.api.Partition, Partition>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/5971015e/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java
index a9db135..58df665 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java
@@ -87,9 +87,13 @@ class AlterPartitionHandler extends AbstractEventHandler {
public void handle(Context withinContext) throws Exception {
LOG.info("Processing#{} ALTER_PARTITION message : {}", fromEventId(), event.getMessage());
+ Table qlMdTable = new Table(tableObject);
+ if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) {
+ return;
+ }
+
if (Scenario.ALTER == scenario) {
withinContext.replicationSpec.setIsMetadataOnly(true);
- Table qlMdTable = new Table(tableObject);
List<Partition> partitions = new ArrayList<>();
partitions.add(new Partition(qlMdTable, after));
Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);
http://git-wip-us.apache.org/repos/asf/hive/blob/5971015e/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
index ab9a9de..4e3ce0e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
class AlterTableHandler extends AbstractEventHandler {
+ private final org.apache.hadoop.hive.metastore.api.Table before;
private final org.apache.hadoop.hive.metastore.api.Table after;
private final boolean isTruncateOp;
private final Scenario scenario;
@@ -58,7 +59,7 @@ class AlterTableHandler extends AbstractEventHandler {
AlterTableHandler(NotificationEvent event) throws Exception {
super(event);
AlterTableMessage atm = deserializer.getAlterTableMessage(event.getMessage());
- org.apache.hadoop.hive.metastore.api.Table before = atm.getTableObjBefore();
+ before = atm.getTableObjBefore();
after = atm.getTableObjAfter();
isTruncateOp = atm.getIsTruncateOp();
scenario = scenarioType(before, after);
@@ -76,23 +77,28 @@ class AlterTableHandler extends AbstractEventHandler {
@Override
public void handle(Context withinContext) throws Exception {
- {
- LOG.info("Processing#{} ALTER_TABLE message : {}", fromEventId(), event.getMessage());
- if (Scenario.ALTER == scenario) {
- withinContext.replicationSpec.setIsMetadataOnly(true);
- Table qlMdTableAfter = new Table(after);
- Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);
- EximUtil.createExportDump(
- metaDataPath.getFileSystem(withinContext.hiveConf),
- metaDataPath,
- qlMdTableAfter,
- null,
- withinContext.replicationSpec);
- }
- DumpMetaData dmd = withinContext.createDmd(this);
- dmd.setPayload(event.getMessage());
- dmd.write();
+ LOG.info("Processing#{} ALTER_TABLE message : {}", fromEventId(), event.getMessage());
+
+ Table qlMdTableBefore = new Table(before);
+ if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTableBefore)) {
+ return;
+ }
+
+ if (Scenario.ALTER == scenario) {
+ withinContext.replicationSpec.setIsMetadataOnly(true);
+ Table qlMdTableAfter = new Table(after);
+ Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);
+ EximUtil.createExportDump(
+ metaDataPath.getFileSystem(withinContext.hiveConf),
+ metaDataPath,
+ qlMdTableAfter,
+ null,
+ withinContext.replicationSpec);
}
+
+ DumpMetaData dmd = withinContext.createDmd(this);
+ dmd.setPayload(event.getMessage());
+ dmd.write();
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/5971015e/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java
index 956bb08..df852a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java
@@ -43,6 +43,11 @@ class InsertHandler extends AbstractEventHandler {
public void handle(Context withinContext) throws Exception {
InsertMessage insertMsg = deserializer.getInsertMessage(event.getMessage());
org.apache.hadoop.hive.ql.metadata.Table qlMdTable = tableObject(insertMsg);
+
+ if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) {
+ return;
+ }
+
List<Partition> qlPtns = null;
if (qlMdTable.isPartitioned() && (null != insertMsg.getPtnObj())) {
qlPtns = Collections.singletonList(partitionObject(qlMdTable, insertMsg));