[GitHub] [hive] sankarh commented on a change in pull request #679: HIVE-21764 : REPL DUMP should detect and bootstrap any rename table events where old table was excluded but renamed table is included.

[GitBox <gi...@apache.org>]

sankarh commented on a change in pull request #679: HIVE-21764 : REPL DUMP should detect and bootstrap any rename table events where old table was excluded but renamed table is included.
URL: https://github.com/apache/hive/pull/679#discussion_r296097381
 
 

 ##########
 File path: itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
 ##########
 @@ -585,4 +585,143 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws
             .run("show tables")
             .verifyResults(incrementalReplicatedTables);
   }
+
+  @Test
+  public void testRenameTableScenariosBasic() throws Throwable {
+    String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+    String lastReplId = replicateAndVerify(replPolicy, null, null, null,
+            null, new String[]{}, new String[]{});
+
+    String[] originalNonAcidTables = new String[]{"in1", "in2", "out3", "out4", "out5", "out6"};
+    createTables(originalNonAcidTables, CreateTableType.NON_ACID);
+
+    // Replicate and verify if only 2 tables are replicated to target.
+    String[] replicatedTables = new String[]{"in1", "in2"};
+    String[] bootstrapTables = new String[]{};
+    lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null,
+            null, bootstrapTables, replicatedTables);
+
+    // Rename tables to make them satisfy the filter.
+    primary.run("use " + primaryDbName)
+            .run("alter table out3 rename to in3")
+            .run("alter table out4 rename to in4")
+            .run("alter table out5 rename to in5");
+
+    replicatedTables = new String[]{"in1", "in2", "in3", "in4", "in5"};
+    bootstrapTables = new String[]{"in3", "in4", "in5"};
+    lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null,
+            null, bootstrapTables, replicatedTables);
+
+    primary.run("use " + primaryDbName)
+            .run("alter table in3 rename to in7")
+            .run("alter table in7 rename to in8") // Double rename, both satisfying the filter, so no bootstrap.
+            .run("alter table in4 rename to out9") // out9 does not match the filter so in4 should be dropped.
+            .run("alter table in5 rename to out10") // Rename from satisfying name to not satisfying name.
+            .run("alter table out10 rename to in11");// from non satisfying to satisfying, should be bootstrapped
+
+    replicatedTables = new String[]{"in1", "in2", "in8", "in11"};
+    bootstrapTables = new String[]{"in11"};
+    lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null,
+            null, bootstrapTables, replicatedTables);
+
+    primary.run("use " + primaryDbName)
+            .run("alter table in8 rename to in12") // table is renamed from satisfying to satisfying, no bootstrap
+            .run("alter table out9 rename to in13") // out9 does not match the filter so in13 should be bootstrapped.
+            .run("alter table in13 rename to in14") // table is renamed from satisfying to satisfying
+            .run("drop table in14");  // table in14 is dropped, so no bootstrap should happen.
+
+    replicatedTables = new String[]{"in1", "in2", "in12", "in12"};
+    bootstrapTables = new String[]{};
+    replicateAndVerify(replPolicy, null, lastReplId, null,
+            null, bootstrapTables, replicatedTables);
+  }
+
+  @Test
+  public void testRenameTableScenariosAcidTable() throws Throwable {
+    String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+    List<String> dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'",
+            "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
+    String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause,
+            null, new String[]{}, new String[]{});
+
+    String[] originalNonAcidTables = new String[]{"in1", "out4"};
+    String[] originalFullAcidTables = new String[]{"in2", "out5"};
+    String[] originalMMAcidTables = new String[]{"out3", "out6"};
+    createTables(originalNonAcidTables, CreateTableType.NON_ACID);
+    createTables(originalFullAcidTables, CreateTableType.FULL_ACID);
+    createTables(originalMMAcidTables, CreateTableType.MM_ACID);
+
+    // Replicate and verify if only 1 tables are replicated to target. Acid tables are not dumped.
+    String[] replicatedTables = new String[]{"in1"};
+    String[] bootstrapTables = new String[]{};
+    lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+            null, bootstrapTables, replicatedTables);
+
+    // Rename tables to make them satisfy the filter and enable acid tables.
+    primary.run("use " + primaryDbName)
+            .run("alter table out3 rename to in3")
+            .run("alter table out4 rename to in4")
+            .run("alter table out5 rename to in5");
+
+    dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='true'",
+            "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'");
+    replicatedTables = new String[]{"in1", "in2", "in3", "in4", "in5"};
+    bootstrapTables = new String[]{"in2", "in3", "in4", "in5"};
+    replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+            null, bootstrapTables, replicatedTables);
+  }
+
+  @Test
+  public void testRenameTableScenariosExternalTable() throws Throwable {
+    String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+    List<String> loadWithClause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica);
+    List<String> dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'",
+            "'" +  HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'",
+            "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'",
+            "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
+    String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause,
+            loadWithClause, new String[]{}, new String[]{});
+
+    String[] originalNonAcidTables = new String[]{"in1", "out4"};
+    String[] originalExternalTables = new String[]{"in2", "out5"};
+    String[] originalMMAcidTables = new String[]{"in3", "out6"};
+    createTables(originalNonAcidTables, CreateTableType.NON_ACID);
+    createTables(originalExternalTables, CreateTableType.EXTERNAL);
+    createTables(originalMMAcidTables, CreateTableType.MM_ACID);
+
+    // Replicate and verify if only 1 tables are replicated to target. Acid and external tables are not dumped.
+    String[] replicatedTables = new String[]{"in1"};
+    String[] bootstrapTables = new String[]{};
+    lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+            loadWithClause, bootstrapTables, replicatedTables);
+
+    // Rename tables to make them satisfy the filter and enable acid and external tables.
+    primary.run("use " + primaryDbName)
+            .run("alter table out4 rename to in4")
+            .run("alter table out5 rename to in5");
+
+    dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
+            "'" +  HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'",
+            "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='true'",
+            "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'");
+    replicatedTables = new String[]{"in1", "in2", "in3", "in4", "in5"};
+    bootstrapTables = new String[]{"in2", "in3", "in4", "in5"};
+    lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+            loadWithClause, null, replicatedTables);
+
+    dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
+            "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'");
+
+    primary.run("use " + primaryDbName)
+            .run("alter table out6 rename to in6")  // external table bootstrap.
+            .run("alter table in5 rename to out7") // in5 should be deleted.
+            .run("alter table out7 rename to in7") // MM table bootstrap.
+            .run("alter table in1 rename to out10") // in1 should be deleted.
+            .run("alter table out10 rename to in11");// normal table bootstrapped
+
+    replicatedTables = new String[]{"in2", "in3", "in4", "in11", "in6", "in7"};
+    bootstrapTables = new String[]{"in11", "in6", "in7"};
+    replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause,
+            loadWithClause, bootstrapTables, replicatedTables);
+  }
 
 Review comment:
   Can add one more scenario as follows?
   1. Replicate with policy= ['in[0-9]+'] which includes table (in1).
   2. Rename (in1 to out1).
   3. Perform some writes into out1.
   4. Rename (out1 to in1).
   5. Incremental replication. Need to check if the data written on out1 is seen after replication or not.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org