You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by an...@apache.org on 2020/08/05 03:45:09 UTC
[hive] branch master updated: HIVE-23961:Enable external table
replication by default (Aasha Medhi, reviewed by Pravin Kumar Sinha)
This is an automated email from the ASF dual-hosted git repository.
anishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5bc50f6 HIVE-23961:Enable external table replication by default (Aasha Medhi, reviewed by Pravin Kumar Sinha)
5bc50f6 is described below
commit 5bc50f6afda439002f63c77fa7e013f64279e4b0
Author: Anishek Agarwal <an...@gmail.com>
AuthorDate: Wed Aug 5 09:14:56 2020 +0530
HIVE-23961:Enable external table replication by default (Aasha Medhi, reviewed by Pravin Kumar Sinha)
---
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +++---
.../hive/metastore/TestMetaStoreMultipleEncryptionZones.java | 8 ++++----
.../org/apache/hadoop/hive/metastore/TestReplChangeManager.java | 8 ++++----
.../java/org/apache/hadoop/hive/ql/parse/TestExportImport.java | 1 +
.../hive/ql/parse/TestReplicationScenariosAcrossInstances.java | 4 ++--
ql/src/test/queries/clientpositive/exim_10_external_managed.q | 1 +
.../java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 2 +-
7 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bba2656..9b637fb 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -511,8 +511,8 @@ public class HiveConf extends Configuration {
"Turn on ChangeManager, so delete files will go to cmrootdir."),
REPLCMDIR("hive.repl.cmrootdir","/user/${system:user.name}/cmroot/",
"Root dir for ChangeManager, used for deleted files."),
- REPLCMRETIAN("hive.repl.cm.retain","24h",
- new TimeValidator(TimeUnit.HOURS),
+ REPLCMRETIAN("hive.repl.cm.retain","7d",
+ new TimeValidator(TimeUnit.DAYS),
"Time to retain removed files in cmrootdir."),
REPLCMENCRYPTEDDIR("hive.repl.cm.encryptionzone.rootdir", ".cmroot",
"Root dir for ChangeManager if encryption zones are enabled, used for deleted files."),
@@ -571,7 +571,7 @@ public class HiveConf extends Configuration {
REPL_ADD_RAW_RESERVED_NAMESPACE("hive.repl.add.raw.reserved.namespace", false,
"For TDE with same encryption keys on source and target, allow Distcp super user to access \n"
+ "the raw bytes from filesystem without decrypting on source and then encrypting on target."),
- REPL_INCLUDE_EXTERNAL_TABLES("hive.repl.include.external.tables", false,
+ REPL_INCLUDE_EXTERNAL_TABLES("hive.repl.include.external.tables", true,
"Indicates if repl dump should include information about external tables. It should be \n"
+ "used in conjunction with 'hive.repl.dump.metadata.only' set to false. if 'hive.repl.dump.metadata.only' \n"
+ " is set to true then this config parameter has no effect as external table meta data is flushed \n"
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
index 8f1bc40..e46a520 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java
@@ -1316,16 +1316,16 @@ public class TestMetaStoreMultipleEncryptionZones {
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11,
ReplChangeManager.getInstance(conf).getCmRoot(part11).toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21,
ReplChangeManager.getInstance(conf).getCmRoot(part21).toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31,
ReplChangeManager.getInstance(conf).getCmRoot(part31).toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32,
ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
ReplChangeManager.scheduleCMClearer(hiveConfCmClearer);
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
index d3891cf..9b728b8 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
@@ -310,13 +310,13 @@ public class TestReplChangeManager {
assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32, cmroot.toString())));
fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11, cmroot.toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21, cmroot.toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31, cmroot.toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32, cmroot.toString()),
- now - 86400*1000*2, now - 86400*1000*2);
+ now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2);
ReplChangeManager.scheduleCMClearer(hiveConf);
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java
index 6f5c8a9..3f4cacf 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java
@@ -54,6 +54,7 @@ public class TestExportImport {
Configuration conf = new Configuration();
conf.set("dfs.client.use.datanode.hostname", "true");
conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*");
+ conf.set("hive.repl.include.external.tables", "false");
MiniDFSCluster miniDFSCluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
HashMap<String, String> overridesForHiveConf = new HashMap<String, String>() {{
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 2953c22..b4666ba 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1000,7 +1000,7 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro
"clustered by(key) into 2 buckets stored as orc")
.run("create table table4 (i int, j int)")
.run("insert into table4 values (1,2)")
- .dump(primaryDbName, Collections.emptyList());
+ .dump(primaryDbName, Collections.singletonList("'hive.repl.include.external.tables'='false'"));
String hiveDumpDir = incremental.dumpLocation + File.separator + ReplUtils.REPL_HIVE_BASE_DIR;
Path path = new Path(hiveDumpDir);
@@ -1009,7 +1009,7 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro
int numEvents = fileStatus.length - 3; //for _metadata, _finished_dump and _events_dump
replica.load(replicatedDbName, primaryDbName,
- Collections.singletonList("'hive.repl.approx.max.load.tasks'='1'"))
+ Arrays.asList("'hive.repl.approx.max.load.tasks'='1','hive.repl.include.external.tables'='false'"))
.run("use " + replicatedDbName)
.run("show tables")
.verifyResults(new String[] {"table1", "table2", "table3", "table4", "table5" })
diff --git a/ql/src/test/queries/clientpositive/exim_10_external_managed.q b/ql/src/test/queries/clientpositive/exim_10_external_managed.q
index 5aadbb3..cda243f 100644
--- a/ql/src/test/queries/clientpositive/exim_10_external_managed.q
+++ b/ql/src/test/queries/clientpositive/exim_10_external_managed.q
@@ -1,4 +1,5 @@
set hive.test.mode=true;
+set hive.repl.include.external.tables=false;
set hive.test.mode.prefix=;
set hive.test.mode.nosamplelist=exim_department_n4,exim_employee;
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 0c43545..b6121a5 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -597,7 +597,7 @@ public class MetastoreConf {
+ "present in HMS Notification. Any key-value pair whose key is matched with any regex will"
+" be removed from Parameters map during Serialization of Table/Partition object."),
EVENT_DB_LISTENER_TTL("metastore.event.db.listener.timetolive",
- "hive.metastore.event.db.listener.timetolive", 86400, TimeUnit.SECONDS,
+ "hive.metastore.event.db.listener.timetolive", 7, TimeUnit.DAYS,
"time after which events will be removed from the database listener queue"),
EVENT_CLEAN_MAX_EVENTS("metastore.event.db.clean.maxevents",
"hive.metastore.event.db.clean.maxevents", 10000,