You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2020/02/19 03:19:02 UTC
[hive] branch master updated: HIVE-22860 : Support metadata only
replication for external tables. (Aasha Medhi,
reviewed by Mahesh Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e0a0db3 HIVE-22860 : Support metadata only replication for external tables. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
e0a0db3 is described below
commit e0a0db37fd6823ba3c3ff3374d775d3d151732fa
Author: Aasha Medhi <aa...@gmail.com>
AuthorDate: Wed Feb 19 08:46:28 2020 +0530
HIVE-22860 : Support metadata only replication for external tables. (Aasha Medhi, reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera <ma...@apache.org>
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../TestReplicationScenariosExternalTables.java | 1 -
...cationScenariosExternalTablesMetaDataOnly.java} | 374 +++------------------
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 9 +-
.../hive/ql/exec/repl/ReplExternalTables.java | 3 +-
.../hive/ql/parse/ReplicationSemanticAnalyzer.java | 4 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 10 +
.../hive/ql/parse/repl/dump/HiveWrapper.java | 2 +-
.../hive/ql/parse/repl/dump/TableExport.java | 12 +-
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 8 +
.../parse/repl/dump/events/AlterTableHandler.java | 2 +-
.../parse/repl/dump/events/CreateTableHandler.java | 3 +-
12 files changed, 91 insertions(+), 340 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d3cb60b..0eee582 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -478,6 +478,9 @@ public class HiveConf extends Configuration {
REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false,
"Indicates whether replication dump only metadata information or data + metadata. \n"
+ "This config makes hive.repl.include.external.tables config ineffective."),
+ REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE("hive.repl.dump.metadata.only.for.external.table",
+ false,
+ "Indicates whether external table replication dump only metadata information or data + metadata"),
REPL_BOOTSTRAP_ACID_TABLES("hive.repl.bootstrap.acid.tables", false,
"Indicates if repl dump should bootstrap the information about ACID tables along with \n"
+ "incremental dump for replication. It is recommended to keep this config parameter \n"
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index 7a90dcc..200e02c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArg
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
-import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
similarity index 62%
copy from itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
copy to itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
index 7a90dcc..1d82467 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java
@@ -24,14 +24,10 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore;
import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection;
-import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArguments;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
-import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables;
import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
@@ -40,36 +36,39 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
import static org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.FILE_NAME;
import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.INC_BOOTSTRAP_ROOT_DIR_NAME;
-import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.REPL_CLEAN_TABLES_FROM_BOOTSTRAP_CONFIG;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-public class TestReplicationScenariosExternalTables extends BaseReplicationAcrossInstances {
+/**
+ * TestMetadataReplicationScenariosExternalTables - Test metadata only replication.
+ * for external tables.
+ */
+public class TestReplicationScenariosExternalTablesMetaDataOnly extends BaseReplicationAcrossInstances {
private static final String REPLICA_EXTERNAL_BASE = "/replica_external_base";
String extraPrimaryDb;
@BeforeClass
public static void classLevelSetup() throws Exception {
- HashMap<String, String> overrides = new HashMap<>();
+ Map<String, String> overrides = new HashMap<>();
overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
GzipJSONMessageEncoder.class.getCanonicalName());
overrides.put(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY.varname, "false");
overrides.put(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname, "true");
+ overrides.put(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE.varname, "true");
overrides.put(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER.varname,
UserGroupInformation.getCurrentUser().getUserName());
@@ -91,9 +90,8 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
@Test
public void replicationWithoutExternalTables() throws Throwable {
List<String> loadWithClause = externalTableBasePathWithClause();
- List<String> dumpWithClause = Collections.singletonList
- ("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'");
-
+ List<String> dumpWithClause = Arrays.asList("'"
+ + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'");
WarehouseInstance.Tuple tuple = primary
.run("use " + primaryDbName)
.run("create external table t1 (id int)")
@@ -114,9 +112,9 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.verifyResult(tuple.lastReplicationId)
.run("use " + replicatedDbName)
.run("show tables like 't1'")
- .verifyFailure(new String[] { "t1" })
+ .verifyFailure(new String[] {"t1"})
.run("show tables like 't2'")
- .verifyFailure(new String[] { "t2" })
+ .verifyFailure(new String[] {"t2"})
.verifyReplTargetProperty(replicatedDbName);
tuple = primary.run("use " + primaryDbName)
@@ -125,14 +123,14 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("insert into table t3 values (20)")
.dump(primaryDbName, tuple.lastReplicationId, dumpWithClause);
- // the _external_tables_file info only should be created if external tables are to be replicated not otherwise
+ // the _external_tables_file data only should be created if external tables are to be replicated not otherwise
assertFalse(primary.miniDFSCluster.getFileSystem()
.exists(new Path(tuple.dumpLocation, FILE_NAME)));
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("show tables like 't3'")
- .verifyFailure(new String[] { "t3" })
+ .verifyFailure(new String[] {"t3"})
.verifyReplTargetProperty(replicatedDbName);
}
@@ -150,9 +148,8 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("insert into table t2 partition(country='france') values ('paris')")
.dump("repl dump " + primaryDbName);
- // verify that the external table info is written correctly for bootstrap
- assertExternalFileInfo(Arrays.asList("t1", "t2"),
- new Path(new Path(tuple.dumpLocation, primaryDbName.toLowerCase()), FILE_NAME));
+ // verify that the external table info is not written as metadata only replication
+ assertFalseExternalFileInfo(new Path(new Path(tuple.dumpLocation, primaryDbName.toLowerCase()), FILE_NAME));
List<String> withClauseOptions = externalTableBasePathWithClause();
@@ -165,16 +162,13 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("repl status " + replicatedDbName)
.verifyResult(tuple.lastReplicationId)
.run("select country from t2 where country = 'us'")
- .verifyResult("us")
+ .verifyResult(null)
.run("select country from t2 where country = 'france'")
- .verifyResult("france");
+ .verifyResult(null);
// Ckpt should be set on bootstrapped db.
replica.verifyIfCkptSet(replicatedDbName, tuple.dumpLocation);
- assertTablePartitionLocation(primaryDbName + ".t1", replicatedDbName + ".t1");
- assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName + ".t2");
-
tuple = primary.run("use " + primaryDbName)
.run("create external table t3 (id int)")
.run("insert into table t3 values (10)")
@@ -182,57 +176,23 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.dump("repl dump " + primaryDbName + " from " + tuple.lastReplicationId);
// verify that the external table info is written correctly for incremental
- assertExternalFileInfo(Arrays.asList("t1", "t2", "t3", "t4"),
- new Path(tuple.dumpLocation, FILE_NAME));
+ assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME));
replica.load(replicatedDbName, tuple.dumpLocation, withClauseOptions)
.run("use " + replicatedDbName)
.run("show tables like 't3'")
.verifyResult("t3")
.run("select id from t3")
- .verifyResult("10")
+ .verifyResult(null)
.run("select id from t4")
- .verifyResult("10");
-
- assertTablePartitionLocation(primaryDbName + ".t3", replicatedDbName + ".t3");
+ .verifyResult(null);
tuple = primary.run("use " + primaryDbName)
.run("drop table t1")
.dump("repl dump " + primaryDbName + " from " + tuple.lastReplicationId);
// verify that the external table info is written correctly for incremental
- assertExternalFileInfo(Arrays.asList("t2", "t3", "t4"),
- new Path(tuple.dumpLocation, FILE_NAME));
- }
-
- /**
- * @param sourceTableName -- Provide the fully qualified table name
- * @param replicaTableName -- Provide the fully qualified table name
- */
- private void assertTablePartitionLocation(String sourceTableName, String replicaTableName)
- throws HiveException {
- Hive hiveForPrimary = Hive.get(primary.hiveConf);
- org.apache.hadoop.hive.ql.metadata.Table sourceTable = hiveForPrimary.getTable(sourceTableName);
- Path sourceLocation = sourceTable.getDataLocation();
- Hive hiveForReplica = Hive.get(replica.hiveConf);
- org.apache.hadoop.hive.ql.metadata.Table replicaTable = hiveForReplica.getTable(replicaTableName);
- Path dataLocation = replicaTable.getDataLocation();
- assertEquals(REPLICA_EXTERNAL_BASE + sourceLocation.toUri().getPath(),
- dataLocation.toUri().getPath());
- if (sourceTable.isPartitioned()) {
- Set<Partition> sourcePartitions = hiveForPrimary.getAllPartitionsOf(sourceTable);
- Set<Partition> replicaPartitions = hiveForReplica.getAllPartitionsOf(replicaTable);
- assertEquals(sourcePartitions.size(), replicaPartitions.size());
- List<String> expectedPaths =
- sourcePartitions.stream()
- .map(p -> REPLICA_EXTERNAL_BASE + p.getDataLocation().toUri().getPath())
- .collect(Collectors.toList());
- List<String> actualPaths =
- replicaPartitions.stream()
- .map(p -> p.getDataLocation().toUri().getPath())
- .collect(Collectors.toList());
- assertTrue(expectedPaths.containsAll(actualPaths));
- }
+ assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME));
}
@Test
@@ -263,8 +223,6 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.verifyResults(Collections.singletonList("a"))
.run("select * From a").verifyResults(Collections.emptyList());
- assertTablePartitionLocation(primaryDbName + ".a", replicatedDbName + ".a");
-
//externally add data to location
try (FSDataOutputStream outputStream =
fs.create(new Path(externalTableLocation, "file1.txt"))) {
@@ -277,9 +235,9 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
replica.load(replicatedDbName, incrementalTuple.dumpLocation, loadWithClause)
.run("select i From a")
- .verifyResults(new String[] { "1", "13" })
+ .verifyResults(new String[] {})
.run("select j from a")
- .verifyResults(new String[] { "2", "21" });
+ .verifyResults(new String[] {});
// alter table location to something new.
externalTableLocation =
@@ -292,7 +250,6 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("use " + replicatedDbName)
.run("select i From a")
.verifyResults(Collections.emptyList());
- assertTablePartitionLocation(primaryDbName + ".a", replicatedDbName + ".a");
}
@Test
@@ -311,19 +268,16 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("insert into t2 partition(country='india') values ('bangalore')")
.dump("repl dump " + primaryDbName);
- assertExternalFileInfo(Collections.singletonList("t2"),
- new Path(new Path(tuple.dumpLocation, primaryDbName.toLowerCase()), FILE_NAME));
+ assertFalseExternalFileInfo(new Path(new Path(tuple.dumpLocation, primaryDbName.toLowerCase()), FILE_NAME));
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("show tables like 't2'")
- .verifyResults(new String[] { "t2" })
+ .verifyResults(new String[] {"t2"})
.run("select place from t2")
- .verifyResults(new String[] { "bangalore" })
+ .verifyResults(new String[] {})
.verifyReplTargetProperty(replicatedDbName);
- assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName + ".t2");
-
// add new data externally, to a partition, but under the table level top directory
Path partitionDir = new Path(externalTableLocation, "country=india");
try (FSDataOutputStream outputStream = fs.create(new Path(partitionDir, "file.txt"))) {
@@ -335,17 +289,16 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("insert into t2 partition(country='australia') values ('sydney')")
.dump(primaryDbName, tuple.lastReplicationId);
- assertExternalFileInfo(Collections.singletonList("t2"),
- new Path(tuple.dumpLocation, FILE_NAME));
+ assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME));
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("select distinct(country) from t2")
- .verifyResults(new String[] { "india", "australia" })
+ .verifyResults(new String[] {})
.run("select place from t2 where country='india'")
- .verifyResults(new String[] { "bangalore", "pune", "mumbai" })
+ .verifyResults(new String[] {})
.run("select place from t2 where country='australia'")
- .verifyResults(new String[] { "sydney" })
+ .verifyResults(new String[] {})
.verifyReplTargetProperty(replicatedDbName);
Path customPartitionLocation =
@@ -366,7 +319,7 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("select place from t2 where country='france'")
- .verifyResults(new String[] { "paris" })
+ .verifyResults(new String[] {})
.verifyReplTargetProperty(replicatedDbName);
// change the location of the partition via alter command
@@ -383,10 +336,6 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.verifyResults(new String[] {})
.verifyReplTargetProperty(replicatedDbName);
- // Changing location of one of the partitions shouldn't result in changing location of other
- // partitions as well as that of the table.
- assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName + ".t2");
-
// Changing location of the external table, should result in changes to the location of
// partition residing within the table location and not the partitions located outside.
String tmpLocation2 = "/tmp/" + System.nanoTime() + "_2";
@@ -398,7 +347,6 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.dump(primaryDbName, tuple.lastReplicationId);
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause);
- assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName + ".t2");
}
@Test
@@ -419,7 +367,7 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("alter table t1 add partition(country='us')")
.dump(primaryDbName, tuple.lastReplicationId);
- assertExternalFileInfo(Collections.singletonList("t1"), new Path(tuple.dumpLocation, FILE_NAME));
+ assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME));
// Add new data externally, to a partition, but under the partition level top directory
// Also, it is added after dumping the events but data should be seen at target after REPL LOAD.
@@ -439,9 +387,9 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.run("show tables like 't1'")
.verifyResult("t1")
.run("show partitions t1")
- .verifyResults(new String[] { "country=india", "country=us" })
+ .verifyResults(new String[] {"country=india", "country=us"})
.run("select place from t1 order by place")
- .verifyResults(new String[] { "bangalore", "mumbai", "pune" })
+ .verifyResults(new String[] {})
.verifyReplTargetProperty(replicatedDbName);
// Delete one of the file and update another one.
@@ -453,16 +401,16 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
// Repl load with zero events but external tables location info should present.
tuple = primary.dump(primaryDbName, tuple.lastReplicationId);
- assertExternalFileInfo(Collections.singletonList("t1"), new Path(tuple.dumpLocation, FILE_NAME));
+ assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME));
replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("show tables like 't1'")
.verifyResult("t1")
.run("show partitions t1")
- .verifyResults(new String[] { "country=india", "country=us" })
+ .verifyResults(new String[] {"country=india", "country=us"})
.run("select place from t1 order by place")
- .verifyResults(new String[] { "chennai" })
+ .verifyResults(new String[] {})
.verifyReplTargetProperty(replicatedDbName);
Hive hive = Hive.get(replica.getConf());
@@ -489,9 +437,8 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
@Test
public void bootstrapExternalTablesDuringIncrementalPhase() throws Throwable {
List<String> loadWithClause = externalTableBasePathWithClause();
- List<String> dumpWithClause = Collections.singletonList(
- "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'"
- );
+ List<String> dumpWithClause
+ = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'");
WarehouseInstance.Tuple tuple = primary
.run("use " + primaryDbName)
@@ -519,7 +466,8 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.verifyReplTargetProperty(replicatedDbName);
dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
- "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'");
+ "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'",
+ "'" + HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE.varname + "'='false'");
tuple = primary.run("use " + primaryDbName)
.run("drop table t1")
.run("create external table t3 (id int)")
@@ -584,132 +532,6 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
}
@Test
- public void retryBootstrapExternalTablesFromDifferentDump() throws Throwable {
- List<String> loadWithClause = new ArrayList<>();
- loadWithClause.addAll(externalTableBasePathWithClause());
-
- List<String> dumpWithClause = Collections.singletonList(
- "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'"
- );
-
- WarehouseInstance.Tuple tupleBootstrapWithoutExternal = primary
- .run("use " + primaryDbName)
- .run("create external table t1 (id int)")
- .run("insert into table t1 values (1)")
- .run("create external table t2 (place string) partitioned by (country string)")
- .run("insert into table t2 partition(country='india') values ('bangalore')")
- .run("insert into table t2 partition(country='us') values ('austin')")
- .run("create table t3 as select * from t1")
- .dump(primaryDbName, null, dumpWithClause);
-
- replica.load(replicatedDbName, tupleBootstrapWithoutExternal.dumpLocation, loadWithClause)
- .status(replicatedDbName)
- .verifyResult(tupleBootstrapWithoutExternal.lastReplicationId)
- .run("use " + replicatedDbName)
- .run("show tables")
- .verifyResult("t3")
- .run("select id from t3")
- .verifyResult("1")
- .verifyReplTargetProperty(replicatedDbName);
-
- dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
- "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'");
- WarehouseInstance.Tuple tupleIncWithExternalBootstrap = primary.run("use " + primaryDbName)
- .run("drop table t1")
- .run("create external table t4 (id int)")
- .run("insert into table t4 values (10)")
- .run("create table t5 as select * from t4")
- .dump(primaryDbName, tupleBootstrapWithoutExternal.lastReplicationId, dumpWithClause);
-
- // Fail setting ckpt property for table t4 but success for t2.
- BehaviourInjection<CallerArguments, Boolean> callerVerifier
- = new BehaviourInjection<CallerArguments, Boolean>() {
- @Nullable
- @Override
- public Boolean apply(@Nullable CallerArguments args) {
- if (args.tblName.equalsIgnoreCase("t4") && args.dbName.equalsIgnoreCase(replicatedDbName)) {
- injectionPathCalled = true;
- LOG.warn("Verifier - DB : " + args.dbName + " TABLE : " + args.tblName);
- return false;
- }
- return true;
- }
- };
-
- // Fail repl load before the ckpt property is set for t4 and after it is set for t2.
- // In the retry, these half baked tables should be dropped and bootstrap should be successful.
- InjectableBehaviourObjectStore.setAlterTableModifier(callerVerifier);
- try {
- replica.loadFailure(replicatedDbName, tupleIncWithExternalBootstrap.dumpLocation, loadWithClause);
- callerVerifier.assertInjectionsPerformed(true, false);
- } finally {
- InjectableBehaviourObjectStore.resetAlterTableModifier();
- }
-
- // Insert into existing external table and then Drop it, add another managed table with same name
- // and dump another bootstrap dump for external tables.
- WarehouseInstance.Tuple tupleNewIncWithExternalBootstrap = primary.run("use " + primaryDbName)
- .run("insert into table t2 partition(country='india') values ('chennai')")
- .run("drop table t2")
- .run("create table t2 as select * from t4")
- .run("insert into table t4 values (20)")
- .dump(primaryDbName, tupleIncWithExternalBootstrap.lastReplicationId, dumpWithClause);
-
- // Set incorrect bootstrap dump to clean tables. Here, used the full bootstrap dump which is invalid.
- // So, REPL LOAD fails.
- loadWithClause.add("'" + REPL_CLEAN_TABLES_FROM_BOOTSTRAP_CONFIG + "'='"
- + tupleBootstrapWithoutExternal.dumpLocation + "'");
- replica.loadFailure(replicatedDbName, tupleNewIncWithExternalBootstrap.dumpLocation, loadWithClause);
- loadWithClause.remove("'" + REPL_CLEAN_TABLES_FROM_BOOTSTRAP_CONFIG + "'='"
- + tupleBootstrapWithoutExternal.dumpLocation + "'");
-
- // Set previously failed bootstrap dump to clean-up. Now, new bootstrap should overwrite the old one.
- loadWithClause.add("'" + REPL_CLEAN_TABLES_FROM_BOOTSTRAP_CONFIG + "'='"
- + tupleIncWithExternalBootstrap.dumpLocation + "'");
-
- // Verify if bootstrapping with same dump is idempotent and return same result
- for (int i = 0; i < 2; i++) {
- replica.load(replicatedDbName, tupleNewIncWithExternalBootstrap.dumpLocation, loadWithClause)
- .run("use " + replicatedDbName)
- .run("show tables like 't1'")
- .verifyFailure(new String[]{"t1"})
- .run("select id from t2")
- .verifyResult("10")
- .run("select id from t4")
- .verifyResults(Arrays.asList("10", "20"))
- .run("select id from t5")
- .verifyResult("10")
- .verifyReplTargetProperty(replicatedDbName);
-
- // Once the REPL LOAD is successful, the this config should be unset or else, the subsequent REPL LOAD
- // will also drop those tables which will cause data loss.
- loadWithClause.remove("'" + REPL_CLEAN_TABLES_FROM_BOOTSTRAP_CONFIG + "'='"
- + tupleIncWithExternalBootstrap.dumpLocation + "'");
- }
- }
-
- @Test
- public void testExternalTableDataPath() throws Exception {
- HiveConf conf = primary.getConf();
- Path basePath = new Path("/");
- Path sourcePath = new Path("/abc/xyz");
- Path dataPath = ReplExternalTables.externalTableDataPath(conf, basePath, sourcePath);
- assertTrue(dataPath.toUri().getPath().equalsIgnoreCase("/abc/xyz"));
-
- basePath = new Path("/tmp");
- dataPath = ReplExternalTables.externalTableDataPath(conf, basePath, sourcePath);
- assertTrue(dataPath.toUri().getPath().equalsIgnoreCase("/tmp/abc/xyz"));
-
- basePath = new Path("/tmp/");
- dataPath = ReplExternalTables.externalTableDataPath(conf, basePath, sourcePath);
- assertTrue(dataPath.toUri().getPath().equalsIgnoreCase("/tmp/abc/xyz"));
-
- basePath = new Path("/tmp/tmp1//");
- dataPath = ReplExternalTables.externalTableDataPath(conf, basePath, sourcePath);
- assertTrue(dataPath.toUri().getPath().equalsIgnoreCase("/tmp/tmp1/abc/xyz"));
- }
-
- @Test
public void testExternalTablesIncReplicationWithConcurrentDropTable() throws Throwable {
List<String> dumpWithClause = Collections.singletonList(
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'"
@@ -754,17 +576,16 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
}
// Only table t2 should exist in the data location list file.
- assertExternalFileInfo(Collections.singletonList("t2"),
- new Path(tupleInc.dumpLocation, FILE_NAME));
+ assertFalseExternalFileInfo(new Path(tupleInc.dumpLocation, FILE_NAME));
// The newly inserted data "2" should be missing in table "t1". But, table t2 should exist and have
// inserted data.
replica.load(replicatedDbName, tupleInc.dumpLocation, loadWithClause)
.run("use " + replicatedDbName)
.run("select id from t1 order by id")
- .verifyResult("1")
+ .verifyResult(null)
.run("select id from t2 order by id")
- .verifyResults(Arrays.asList("1", "2"));
+ .verifyResults(new String[]{});
}
@Test
@@ -806,107 +627,18 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
.verifyResult(inc2Tuple.lastReplicationId);
}
- @Test
- public void testExtTableBootstrapDuringIncrementalWithoutAnyEvents() throws Throwable {
- List<String> loadWithClause = externalTableBasePathWithClause();
- List<String> dumpWithClause = Collections.singletonList(
- "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'"
- );
-
- WarehouseInstance.Tuple bootstrapDump = primary
- .run("use " + primaryDbName)
- .run("create external table t1 (id int)")
- .run("insert into table t1 values (1)")
- .run("create table t2 (id int)")
- .run("insert into table t2 values (1)")
- .dump(primaryDbName, null, dumpWithClause);
-
- replica.load(replicatedDbName, bootstrapDump.dumpLocation, loadWithClause)
- .status(replicatedDbName)
- .verifyResult(bootstrapDump.lastReplicationId)
- .run("use " + replicatedDbName)
- .run("show tables like 't1'")
- .verifyFailure(new String[] {"t1" })
- .run("show tables like 't2'")
- .verifyResult("t2")
- .verifyReplTargetProperty(replicatedDbName);
-
- // This looks like an empty dump but it has the ALTER TABLE event created by the previous
- // dump. We need it here so that the next dump won't have any events.
- WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, bootstrapDump.lastReplicationId);
- replica.load(replicatedDbName, incTuple.dumpLocation)
- .status(replicatedDbName)
- .verifyResult(incTuple.lastReplicationId);
-
- // Take a dump with external tables bootstrapped and load it
- dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
- "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'");
- WarehouseInstance.Tuple inc2Tuple = primary.run("use " + primaryDbName)
- .dump(primaryDbName, incTuple.lastReplicationId, dumpWithClause);
-
- replica.load(replicatedDbName, inc2Tuple.dumpLocation, loadWithClause)
- .status(replicatedDbName)
- .verifyResult(inc2Tuple.lastReplicationId)
- .run("use " + replicatedDbName)
- .run("show tables like 't1'")
- .verifyResult("t1")
- .run("show tables like 't2'")
- .verifyResult("t2")
- .verifyReplTargetProperty(replicatedDbName);
- }
-
- @Test
- public void replicationWithTableNameContainsKeywords() throws Throwable {
- List<String> loadWithClause = externalTableBasePathWithClause();
-
- WarehouseInstance.Tuple tuple = primary
- .run("use " + primaryDbName)
- .run("create external table t1_functions (id int)")
- .run("insert into table t1_functions values (1)")
- .run("insert into table t1_functions values (2)")
- .run("create external table t2_constraints (place string) partitioned by (country string)")
- .run("insert into table t2_constraints partition(country='india') values ('bangalore')")
- .run("insert into table t2_constraints partition(country='us') values ('austin')")
- .run("insert into table t2_constraints partition(country='france') values ('paris')")
- .dump(primaryDbName, null);
-
- replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
- .run("repl status " + replicatedDbName)
- .verifyResult(tuple.lastReplicationId)
- .run("use " + replicatedDbName)
- .run("show tables like 't1_functions'")
- .verifyResults(new String[] {"t1_functions"})
- .run("show tables like 't2_constraints'")
- .verifyResults(new String[] {"t2_constraints"})
- .run("select id from t1_functions")
- .verifyResults(new String[] {"1", "2"})
- .verifyReplTargetProperty(replicatedDbName);
-
- tuple = primary.run("use " + primaryDbName)
- .run("create external table t3_bootstrap (id int)")
- .run("insert into table t3_bootstrap values (10)")
- .run("insert into table t3_bootstrap values (20)")
- .run("create table t4_tables (id int)")
- .run("insert into table t4_tables values (10)")
- .run("insert into table t4_tables values (20)")
- .dump(primaryDbName, tuple.lastReplicationId);
-
- replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause)
- .run("use " + replicatedDbName)
- .run("show tables like 't3_bootstrap'")
- .verifyResults(new String[] {"t3_bootstrap"})
- .run("show tables like 't4_tables'")
- .verifyResults(new String[] {"t4_tables"})
- .verifyReplTargetProperty(replicatedDbName);
- }
-
-
private List<String> externalTableBasePathWithClause() throws IOException, SemanticException {
return ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica);
}
- private void assertExternalFileInfo(List<String> expected, Path externalTableInfoFile)
+ private void assertFalseExternalFileInfo(Path externalTableInfoFile)
throws IOException {
+ DistributedFileSystem fileSystem = primary.miniDFSCluster.getFileSystem();
+ Assert.assertFalse(fileSystem.exists(externalTableInfoFile));
+ }
+
+ private void assertExternalFileInfo(List<String> expected, Path externalTableInfoFile)
+ throws IOException {
ReplicationTestUtils.assertExternalFileInfo(primary, expected, externalTableInfoFile);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 622433b..977abb7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -172,7 +172,8 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
*/
private boolean shouldDumpExternalTableLocation() {
return conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES)
- && !conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY);
+ && (!conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY) &&
+ !conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE));
}
/**
@@ -491,9 +492,6 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
String uniqueKey = Utils.setDbBootstrapDumpState(hiveDb, dbName);
Exception caught = null;
- boolean shouldWriteExternalTableLocationInfo =
- conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES)
- && !conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY);
try (Writer writer = new Writer(dbRoot, conf)) {
for (String tblName : Utils.matchesTbl(hiveDb, dbName, work.replScope)) {
LOG.debug("Dumping table: " + tblName + " to db root " + dbRoot.toUri());
@@ -502,7 +500,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
try {
HiveWrapper.Tuple<Table> tableTuple = new HiveWrapper(hiveDb, dbName).table(tblName, conf);
table = tableTuple != null ? tableTuple.object : null;
- if (shouldWriteExternalTableLocationInfo
+ if (shouldDumpExternalTableLocation()
&& TableType.EXTERNAL_TABLE.equals(tableTuple.object.getTableType())) {
LOG.debug("Adding table {} to external tables list", tblName);
writer.dataLocationDump(tableTuple.object);
@@ -583,6 +581,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
tuple.replicationSpec.setCurrentReplicationState(String.valueOf(lastReplId));
}
MmContext mmCtx = MmContext.createIfNeeded(tableSpec.tableHandle);
+ tuple.replicationSpec.setRepl(true);
new TableExport(
exportPaths, tableSpec, tuple.replicationSpec, hiveDb, distCpDoAsUser, conf, mmCtx).write();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java
index 4c504be..c7aa007 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java
@@ -106,7 +106,8 @@ public final class ReplExternalTables {
this.hiveConf = hiveConf;
writePath = new Path(dbRoot, FILE_NAME);
includeExternalTables = hiveConf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES);
- dumpMetadataOnly = hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY);
+ dumpMetadataOnly = hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY) ||
+ hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE);
if (shouldWrite()) {
this.writer = FileSystem.get(hiveConf).create(writePath);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 276f759..810a4c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -49,12 +49,12 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.Reader;
-import static org.apache.hadoop.hive.ql.exec.repl.ExternalTableCopyTaskBuilder.DirCopyWork;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEQUERYID;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_ENABLE_MOVE_OPTIMIZATION;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_MOVE_OPTIMIZED_FILE_SCHEMES;
+import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.Reader;
+import static org.apache.hadoop.hive.ql.exec.repl.ExternalTableCopyTaskBuilder.DirCopyWork;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
index ad3e55a..99b09e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
@@ -51,6 +51,8 @@ public class ReplicationSpec {
private boolean isMigratingToTxnTable = false;
private boolean isMigratingToExternalTable = false;
private boolean needDupCopyCheck = false;
+ //Determine if replication is done using repl or export-import
+ private boolean isRepl = false;
// Key definitions related to replication.
public enum KEY {
@@ -438,4 +440,12 @@ public class ReplicationSpec {
// Check HIVE-21197 for more detail.
this.needDupCopyCheck = isFirstIncPending;
}
+
+ public boolean isRepl() {
+ return isRepl;
+ }
+
+ public void setRepl(boolean repl) {
+ isRepl = repl;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapper.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapper.java
index d01e24c..f9648c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapper.java
@@ -57,7 +57,7 @@ public class HiveWrapper {
public Tuple<Table> table(final String tableName, HiveConf conf) throws HiveException {
// Column statistics won't be accurate if we are dumping only metadata
- boolean getColStats = !conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY);
+ boolean getColStats = !Utils.shouldDumpMetaDataOnly(db.getTable(dbName, tableName), conf);
return new Tuple<>(functionForSpec, () -> db.getTable(dbName, tableName, true, false,
getColStats));
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 01b7fdc..97a1dd3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
@@ -72,8 +73,8 @@ public class TableExport {
? null
: tableSpec;
this.replicationSpec = replicationSpec;
- if (conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY) ||
- (this.tableSpec != null && this.tableSpec.tableHandle.isView())) {
+ if (this.tableSpec != null && this.tableSpec.tableHandle!=null && (this.tableSpec.tableHandle.isView() ||
+ Utils.shouldDumpMetaDataOnly(this.tableSpec.tableHandle, conf))) {
this.replicationSpec.setIsMetadataOnly(true);
this.tableSpec.tableHandle.setStatsStateLikeNewTable();
@@ -92,7 +93,8 @@ public class TableExport {
} else if (shouldExport()) {
PartitionIterable withPartitions = getPartitions();
writeMetaData(withPartitions);
- if (!replicationSpec.isMetadataOnly()) {
+ if (!replicationSpec.isMetadataOnly()
+ && !(replicationSpec.isRepl() && tableSpec.tableHandle.getTableType().equals(TableType.EXTERNAL_TABLE))) {
writeData(withPartitions);
}
return true;
@@ -158,10 +160,8 @@ public class TableExport {
} else {
List<Path> dataPathList = Utils.getDataPathList(tableSpec.tableHandle.getDataLocation(),
replicationSpec, conf);
-
- // this is the data copy
new FileOperations(dataPathList, paths.dataExportDir(), distCpDoAsUser, conf, mmCtx)
- .export(replicationSpec);
+ .export(replicationSpec);
}
} catch (Exception e) {
throw new SemanticException(e.getMessage(), e);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
index bc9f06d..6f8912b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.repl.ReplScope;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -270,4 +271,11 @@ public class Utils {
return Collections.singletonList(fromPath);
}
}
+
+ public static boolean shouldDumpMetaDataOnly(Table table, HiveConf conf) {
+ return conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY) ||
+ (conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES) &&
+ table.getTableType().equals(TableType.EXTERNAL_TABLE) &&
+ conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY_FOR_EXTERNAL_TABLE));
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
index 0168240..aedf698 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java
@@ -229,7 +229,7 @@ class AlterTableHandler extends AbstractEventHandler<AlterTableMessage> {
// If we are not dumping metadata about a table, we shouldn't be dumping basic statistics
// as well, since that won't be accurate. So reset them to what they would look like for an
// empty table.
- if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) {
+ if (Utils.shouldDumpMetaDataOnly(qlMdTableAfter, withinContext.hiveConf)) {
qlMdTableAfter.setStatsStateLikeNewTable();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
index 837d51c..355374a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -66,7 +65,7 @@ class CreateTableHandler extends AbstractEventHandler<CreateTableMessage> {
// If we are not dumping data about a table, we shouldn't be dumping basic statistics
// as well, since that won't be accurate. So reset them to what they would look like for an
// empty table.
- if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) {
+ if (Utils.shouldDumpMetaDataOnly(qlMdTable, withinContext.hiveConf)) {
qlMdTable.setStatsStateLikeNewTable();
}