[hive] branch master updated: HIVE-26960: Fix optimized bootstrap when primary is modified only by addition of new tables (Rakshith Chandraiah, reviewed by Teddy Choi)

[tc...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tchoi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new e20102c6201 HIVE-26960: Fix optimized bootstrap when primary is modified only by addition of new tables (Rakshith Chandraiah, reviewed by Teddy Choi)
e20102c6201 is described below

commit e20102c62015bb74b453a3460352fd789bf9edc7
Author: Rakshith C <56...@users.noreply.github.com>
AuthorDate: Wed Feb 1 06:52:11 2023 +0530

    HIVE-26960: Fix optimized bootstrap when primary is modified only by addition of new tables (Rakshith Chandraiah, reviewed by Teddy Choi)
---
 .../parse/TestReplicationOptimisedBootstrap.java   |  7 ++++--
 .../hadoop/hive/ql/exec/repl/ReplLoadWork.java     | 27 +++++++++++++---------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOptimisedBootstrap.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOptimisedBootstrap.java
index 8d9429e33b8..182cb966dfc 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOptimisedBootstrap.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOptimisedBootstrap.java
@@ -1202,7 +1202,8 @@ public class TestReplicationOptimisedBootstrap extends BaseReplicationScenariosA
 
     // make some changes on primary
     primary.run("use " + primaryDbName)
-            .run("insert into table t1 values (4)");
+            .run("create table t2(name string) stored as orc tblproperties(\"transactional\"=\"true\")")
+            .run("insert into t2 values('a')");
 
     withClause = Arrays.asList(
             String.format("'%s'='%s'", HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false")
@@ -1232,7 +1233,9 @@ public class TestReplicationOptimisedBootstrap extends BaseReplicationScenariosA
     }
     // ensure optimized bootstrap was successful.
     primary.run(String.format("select * from %s.t1", primaryDbName))
-            .verifyResults(new String[]{"1", "2", "3"});
+            .verifyResults(new String[]{"1", "2", "3"})
+            .run("show tables in "+primaryDbName)
+            .verifyResults(new String[]{"t1"});
   }
   @Test
   public void testReverseFailoverBeforeOptimizedBootstrap() throws Throwable {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
index b6072912c93..2c379472d3a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
@@ -159,20 +159,25 @@ public class ReplLoadWork implements Serializable, ReplLoadWorkMBean {
        * for the same.
        */
       Path incBootstrapDir = new Path(dumpDirectory, ReplUtils.INC_BOOTSTRAP_ROOT_DIR_NAME);
-      if (fs.exists(incBootstrapDir)) {
-        if (isSecondFailover) {
-          String[] bootstrappedTables = getBootstrapTableList(new Path(dumpDirectory).getParent(), hiveConf);
-          LOG.info("Optimised bootstrap load for database {} with initial bootstrapped table list as {}",
-              dbNameToLoadIn, tablesToBootstrap);
-          // Get list of tables bootstrapped.
+      if (isSecondFailover) {
+        String[] bootstrappedTables = getBootstrapTableList(new Path(dumpDirectory).getParent(), hiveConf);
+        LOG.info("Optimised bootstrap load for database {} with initial bootstrapped table list as {}",
+                dbNameToLoadIn, tablesToBootstrap);
+        // Get list of tables bootstrapped.
+        if (fs.exists(incBootstrapDir)) {
           Path tableMetaPath = new Path(incBootstrapDir, EximUtil.METADATA_PATH_NAME + "/" + sourceDbName);
           tablesToBootstrap =
-              Stream.of(fs.listStatus(tableMetaPath)).map(st -> st.getPath().getName()).collect(Collectors.toList());
-          List<String> tableList = Arrays.asList(bootstrappedTables);
-          tablesToDrop = ListUtils.subtract(tableList, tablesToBootstrap);
-          LOG.info("Optimised bootstrap for database {} with drop table list as {} and bootstrap table list as {}",
-              dbNameToLoadIn, tablesToDrop, tablesToBootstrap);
+                  Stream.of(fs.listStatus(tableMetaPath)).map(st -> st.getPath().getName()).collect(Collectors.toList());
+        }
+        else {
+          tablesToBootstrap = Collections.emptyList();
         }
+        List<String> tableList = Arrays.asList(bootstrappedTables);
+        tablesToDrop = ListUtils.subtract(tableList, tablesToBootstrap);
+        LOG.info("Optimised bootstrap for database {} with drop table list as {} and bootstrap table list as {}",
+                dbNameToLoadIn, tablesToDrop, tablesToBootstrap);
+      }
+      if (fs.exists(incBootstrapDir)) {
         this.bootstrapIterator = new BootstrapEventsIterator(
                 new Path(incBootstrapDir, EximUtil.METADATA_PATH_NAME).toString(), dbNameToLoadIn, true,
           hiveConf, metricCollector);