You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2019/07/10 03:41:15 UTC
[hive] branch master updated: HIVE-21958 : The list of table
expression in the inclusion and exclusion list should be separated by '|'
instead of comma. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e2a8375 HIVE-21958 : The list of table expression in the inclusion and exclusion list should be separated by '|' instead of comma. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
e2a8375 is described below
commit e2a8375fe3e8e3c601c770556b104bb47debd33e
Author: mbehera <ma...@apache.org>
AuthorDate: Wed Jul 10 09:09:20 2019 +0530
HIVE-21958 : The list of table expression in the inclusion and exclusion list should be separated by '|' instead of comma. (Mahesh Kumar Behera reviewed by Sankar Hariappan)
---
.../parse/TestReplicationWithTableMigrationEx.java | 4 +-
.../parse/TestTableLevelReplicationScenarios.java | 100 +++++++++++----------
.../plugin/TestHiveAuthorizerCheckInvocation.java | 2 +-
.../org/apache/hadoop/hive/ql/parse/HiveParser.g | 18 +---
.../hive/ql/parse/ReplicationSemanticAnalyzer.java | 25 ++----
.../hive/ql/parse/repl/load/DumpMetaData.java | 68 +++++---------
.../ql/parse/TestReplicationSemanticAnalyzer.java | 20 ++---
.../apache/hadoop/hive/common/repl/ReplScope.java | 55 +++++-------
8 files changed, 114 insertions(+), 178 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
index 3a86cb7..f0f1ce5 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java
@@ -221,14 +221,14 @@ public class TestReplicationWithTableMigrationEx {
WarehouseInstance.Tuple tuple = primary
.run("use " + primaryDbName)
.run("create table t1 (i int, j int)")
- .dump(primaryDbName+".['t1']", null);
+ .dump(primaryDbName+".'t1'", null);
replica.run("create database " + replicatedDbName);
replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation);
assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters()));
tuple = primary.run("use " + primaryDbName)
.run("insert into t1 values (1, 2)")
- .dump(primaryDbName+".['t1']", tuple.lastReplicationId);
+ .dump(primaryDbName+".'t1'", tuple.lastReplicationId);
replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation);
assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters()));
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
index a3f56f7..09db38d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java
@@ -204,7 +204,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
- String replPolicy = primaryDbName + ".['t1', 't4', 't5']";
+ String replPolicy = primaryDbName + ".'(t1)|(t4)|(t5)'";
String[] replicatedTables = new String[] {"t1", "t4", "t5"};
replicateAndVerify(replPolicy, null, null, null, replicatedTables);
}
@@ -215,7 +215,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
createTables(originalTables, CreateTableType.NON_ACID);
// Replicate and verify if only 3 tables are replicated to target.
- String replPolicy = primaryDbName + ".['t1*', 't3'].['t100']";
+ String replPolicy = primaryDbName + ".'(t1*)|(t3)'.'t100'";
String[] replicatedTables = new String[] {"t1", "t11", "t3"};
replicateAndVerify(replPolicy, null, null, null, replicatedTables);
}
@@ -234,7 +234,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
- String replPolicy = primaryDbName + ".['t1', 't5']";
+ String replPolicy = primaryDbName + ".'t1|t5'";
String[] replicatedTables = new String[] {"t1", "t5"};
replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables);
}
@@ -249,7 +249,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
createTables(originalTables, CreateTableType.NON_ACID);
// Replicate and verify if only 3 tables are replicated to target.
- String replPolicy = primaryDbName + ".['t1+', 't2'].['t11', 't3']";
+ String replPolicy = primaryDbName + ".'(t1+)|(t2)'.'t11|t3'";
String[] replicatedTables = new String[] {"t1", "t111", "t2"};
replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables);
}
@@ -263,12 +263,13 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
String[] replicatedTables = new String[] {};
boolean failed;
String[] invalidReplPolicies = new String[] {
- primaryDbName + ".t1.t2", // Two explicit table names not allowed.
- primaryDbName + ".['t1'].t2", // Table name and include list not allowed.
- primaryDbName + ".t1.['t2']", // Table name and exclude list not allowed.
- primaryDbName + ".[t1].t2", // Table name and include list not allowed.
- primaryDbName + ".['t1+'].", // Abrubtly ended dot.
- primaryDbName + "..[]" // Multiple dots
+ primaryDbName + ".t1.t2", // Didn't enclose table pattern within single quotes.
+ primaryDbName + ".'t1'.t2", // Table name and include list not allowed.
+ primaryDbName + ".t1.'t2'", // Table name and exclude list not allowed.
+ primaryDbName + ".'t1+'.", // Abrubtly ended dot.
+ primaryDbName + ".['t1+'].['t11']", // With square brackets
+ primaryDbName + "..''", // Two dots with empty list
+ primaryDbName + ".'t1'.'tt2'.'t3'" // More than two list
};
for (String replPolicy : invalidReplPolicies) {
failed = false;
@@ -302,7 +303,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Replace with replication policy having different DB name.
String oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + "_dupe.['t1+'].['t1']";
+ replPolicy = primaryDbName + "_dupe.'t1+'.'t1'";
failed = false;
try {
replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables);
@@ -313,18 +314,23 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
}
Assert.assertTrue(failed);
- // Invalid pattern where we didn't enclose table pattern within single or double quotes.
- replPolicy = primaryDbName + ".[t1].[t2]";
- failed = false;
- try {
- replicateAndVerify(replPolicy, null, null, null, replicatedTables);
- } catch (Exception ex) {
- LOG.info("Got exception: {}", ex.getMessage());
- Assert.assertTrue(ex instanceof SemanticException);
- Assert.assertTrue(ex.getMessage().equals(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg()));
- failed = true;
+ // Invalid pattern, include/exclude table list is empty.
+ invalidReplPolicies = new String[] {
+ primaryDbName + ".''.'t2'", // Include list is empty.
+ primaryDbName + ".'t1'.''" // Exclude list is empty.
+ };
+ for (String invalidReplPolicy : invalidReplPolicies) {
+ failed = false;
+ try {
+ replicateAndVerify(invalidReplPolicy, null, null, null, replicatedTables);
+ } catch (Exception ex) {
+ LOG.info("Got exception: {}", ex.getMessage());
+ Assert.assertTrue(ex instanceof SemanticException);
+ Assert.assertTrue(ex.getMessage().equals(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg()));
+ failed = true;
+ }
+ Assert.assertTrue(failed);
}
- Assert.assertTrue(failed);
}
@Test
@@ -334,8 +340,8 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// List of repl policy formats that leads to Full DB replication.
String[] fullDbReplPolicies = new String[] {
- primaryDbName + ".['.*?']",
- primaryDbName + ".['.*?'].[]"
+ primaryDbName + ".'.*?'",
+ primaryDbName
};
// Replicate and verify if all 3 tables are replicated to target.
@@ -350,13 +356,13 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
createTables(originalTables, CreateTableType.NON_ACID);
// Replicate and verify if 2 tables are replicated as per policy.
- String replPolicy = primaryDbName.toUpperCase() + ".['.*a1+', 'cc3', 'B2'].['AA1+', 'b2']";
+ String replPolicy = primaryDbName.toUpperCase() + ".'(.*a1+)|(cc3)|(B2)'.'(AA1+)|(b2)'";
String[] replicatedTables = new String[] {"a1", "cc3"};
String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables);
// Test case insensitive nature in REPLACE clause as well.
String oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['.*a1+', 'cc3', 'B2'].['AA1+']";
+ replPolicy = primaryDbName + ".'(.*a1+)|(cc3)|(B2)'.'AA1+'";
replicatedTables = new String[] {"a1", "b2", "cc3"};
String[] bootstrappedTables = new String[] {"b2"};
replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables);
@@ -374,7 +380,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Replicate and verify if only non-acid tables are replicated to target.
List<String> dumpWithoutAcidClause = Collections.singletonList(
"'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
- String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a4']";
+ String replPolicy = primaryDbName + ".'(a[0-9]+)|(b1)'.'a4'";
String[] bootstrapReplicatedTables = new String[] {"a1"};
String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables);
@@ -398,7 +404,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
List<String> dumpWithClause = Collections.singletonList(
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'"
);
- String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']";
+ String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'";
String[] replicatedTables = new String[] {"a2", "b2"};
WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName)
.dump(replPolicy, null, dumpWithClause);
@@ -429,7 +435,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
List<String> dumpWithClause = Collections.singletonList(
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'"
);
- String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']";
+ String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'";
String[] bootstrapReplicatedTables = new String[] {"b2"};
String lastReplId = replicateAndVerify(replPolicy, null,
dumpWithClause, loadWithClause, bootstrapReplicatedTables);
@@ -465,7 +471,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
createTables(originalMMAcidTables, CreateTableType.MM_ACID);
// Replicate and verify if only 2 tables are replicated to target.
- String replPolicy = primaryDbName + ".['t1', 't4']";
+ String replPolicy = primaryDbName + ".'(t1)|(t4)'";
String oldReplPolicy = null;
String[] replicatedTables = new String[] {"t1", "t4"};
String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables);
@@ -473,7 +479,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Exclude t4 and include t3, t6
createTables(new String[] {"t6"}, CreateTableType.MM_ACID);
oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['t1', 't3', 't6']";
+ replPolicy = primaryDbName + ".'t1|t3|t6'";
replicatedTables = new String[] {"t1", "t3", "t6"};
String[] bootstrappedTables = new String[] {"t3", "t6"};
lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId,
@@ -489,7 +495,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Convert to regex that excludes t3, t4 and t5.
oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['.*?'].['t[3-5]+']";
+ replPolicy = primaryDbName + ".'.*?'.'t[3-5]+'";
replicatedTables = new String[] {"t1", "t2", "t6"};
bootstrappedTables = new String[] {};
replicateAndVerify(replPolicy, oldReplPolicy, lastReplId,
@@ -508,7 +514,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Replicate and verify if only non-acid tables are replicated to target.
List<String> dumpWithoutAcidClause = Collections.singletonList(
"'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
- String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']";
+ String replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'b1'";
String[] bootstrapReplicatedTables = new String[] {"a1"};
String lastReplId = replicateAndVerify(replPolicy, null,
dumpWithoutAcidClause, null, bootstrapReplicatedTables);
@@ -516,7 +522,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Enable acid tables for replication. Also, replace, replication policy to exclude "b1" and "a3"
// instead of "a1" alone.
String oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['a3', 'b1']";
+ replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'(a3)|(b1)'";
List<String> dumpWithAcidBootstrapClause = Arrays.asList(
"'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'",
"'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'");
@@ -536,7 +542,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Replicate and verify if only non-acid tables are replicated to target.
List<String> dumpWithoutAcidClause = Collections.singletonList(
"'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'");
- String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']";
+ String replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'b1'";
String[] bootstrapReplicatedTables = new String[] {"a1"};
String lastReplId = replicateAndVerify(replPolicy, null,
dumpWithoutAcidClause, null, bootstrapReplicatedTables);
@@ -545,7 +551,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// "a2" but exclude "a1" and "b1". Still ACID tables shouldn't be bootstrapped. Only non-ACID
// table "b1" should be bootstrapped.
String oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2']";
+ replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'a2'";
String[] incrementalReplicatedTables = new String[] {"a1", "b1"};
String[] bootstrappedTables = new String[] {"b1"};
lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId,
@@ -564,7 +570,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
List<String> dumpWithClause = Collections.singletonList(
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'"
);
- String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a1']";
+ String replPolicy = primaryDbName + ".'(a[0-9]+)|(b1)'.'a1'";
String[] bootstrapReplicatedTables = new String[] {"b1"};
String lastReplId = replicateAndVerify(replPolicy, null,
dumpWithClause, loadWithClause, bootstrapReplicatedTables);
@@ -572,7 +578,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Continue to disable external tables for replication. Also, replace, replication policy to exclude
// "b1" and include "a1".
String oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2', 'b1']";
+ replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'(a2)|(b1)'";
String[] incrementalReplicatedTables = new String[] {"a1"};
String[] bootstrappedTables = new String[] {"a1"};
lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId,
@@ -581,7 +587,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
// Enable external tables replication and bootstrap in incremental phase. Also, replace,
// replication policy to exclude tables with prefix "b".
oldReplPolicy = replPolicy;
- replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['b[0-9]+']";
+ replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'b[0-9]+'";
incrementalReplicatedTables = new String[] {"a1", "a2", "c2"};
bootstrappedTables = new String[] {"a2", "c2"};
dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
@@ -608,7 +614,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
@Test
public void testRenameTableScenariosBasic() throws Throwable {
- String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'";
String lastReplId = replicateAndVerify(replPolicy, null, null, null,
null, new String[] {}, new String[] {});
@@ -662,7 +668,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
@Test
public void testRenameTableScenariosWithDmlOperations() throws Throwable {
- String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'";
String lastReplId = replicateAndVerify(replPolicy, null, null, null,
null, new String[] {}, new String[] {});
@@ -694,7 +700,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
@Test
public void testRenameTableScenariosAcidTable() throws Throwable {
- String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'";
List<String> dumpWithClause = Arrays.asList(
"'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'",
"'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"
@@ -731,7 +737,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
@Test
public void testRenameTableScenariosExternalTable() throws Throwable {
- String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']";
+ String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'";
List<String> loadWithClause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica);
List<String> dumpWithClause = Arrays.asList(
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'",
@@ -794,7 +800,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
"'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"
);
- String replPolicy = primaryDbName + ".['in[0-9]+', 'out4', 'out5', 'out1500']";
+ String replPolicy = primaryDbName + ".'(in[0-9]+)|(out4)|(out5)|(out1500)'";
String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause,
loadWithClause, new String[] {}, new String[] {});
@@ -813,7 +819,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
.run("alter table out11 rename to out12") // Old name not matching old and new name not matching both
.run("alter table out12 rename to in12"); // Old name not matching old and new name matching both
- String newPolicy = primaryDbName + ".['in[0-9]+', 'out1500'].['in2']";
+ String newPolicy = primaryDbName + ".'(in[0-9]+)|(out1500)|(in2)'";
dumpWithClause = Arrays.asList(
"'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'",
"'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'"
@@ -828,7 +834,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
@Test
public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Throwable {
- String replPolicy = primaryDbName + ".['in[0-9]+', 'out5000', 'out5001'].['in100', 'in200', 'in305']";
+ String replPolicy = primaryDbName + ".'(in[0-9]+)|(out5000)|(out5001)'.'(in100)|(in200)|(in305)'";
String lastReplId = replicateAndVerify(replPolicy, null, null, null,
null, new String[] {}, new String[] {});
@@ -845,7 +851,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios
null, bootstrapTables, replicatedTables);
// Rename the tables to satisfy the condition also replace the policy.
- String newPolicy = primaryDbName + ".['in[0-9]+', 'out3000'].['in2']";
+ String newPolicy = primaryDbName + ".'(in[0-9]+)|(out3000)'.'in2'";
primary.run("use " + primaryDbName)
.run("alter table in200 rename to in2000") // Old name not matching old, old and new matching new policy.
.run("alter table in400 rename to out400") // Old name matching new and old policy, new matching none.
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index eeda423..5c803ca 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -582,7 +582,7 @@ public class TestHiveAuthorizerCheckInvocation {
assertEquals("db name", dbName.toLowerCase(), dbObj.getDbname());
resetAuthorizer();
- status = driver.compile("repl dump " + dbName + ".['" + inDbTableName + "']");
+ status = driver.compile("repl dump " + dbName + ".'" + inDbTableName + "'");
assertEquals(0, status);
inputs = getHivePrivilegeObjectInputs().getLeft();
dbObj = inputs.get(0);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index a4b233a..3a2a2d0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -939,26 +939,10 @@ replTableLevelPolicy
@init { pushMsg("Replication table level policy definition", state); }
@after { popMsg(state); }
:
- ((replTablesIncludeList=replTablesList) (DOT replTablesExcludeList=replTablesList)?)
+ ((replTablesIncludeList=StringLiteral) (DOT replTablesExcludeList=StringLiteral)?)
-> ^(TOK_REPL_TABLES $replTablesIncludeList $replTablesExcludeList?)
;
-replTablesList
-@init { pushMsg("replication table name or comma separated table names pattern list", state); }
-@after { popMsg(state); }
- :
- (LSQUARE (tablePattern (COMMA tablePattern)*)? RSQUARE) -> ^(TOK_REPL_TABLES_LIST tablePattern*)
- ;
-
-tablePattern
-@init { pushMsg("Table name pattern", state); }
-@after { popMsg(state); }
- :
- (pattern=StringLiteral) -> $pattern
- |
- (identifier) -> TOK_NULL
- ;
-
replStatusStatement
@init { pushMsg("replication status statement", state); }
@after { popMsg(state); }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index df41a2e..39789ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -58,14 +58,12 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_MOVE_OPTIMIZED_
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_NULL;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPLACE;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_CONFIG;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_DUMP;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_LOAD;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_STATUS;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_TABLES;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_TABLES_LIST;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TO;
public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
@@ -140,25 +138,16 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
// and exclude tables lists.
String replScopeType = (replScope == this.replScope) ? "Current" : "Old";
for (int listIdx = 0; listIdx < childCount; listIdx++) {
- Tree tablesListNode = replTablesNode.getChild(listIdx);
- assert(tablesListNode.getType() == TOK_REPL_TABLES_LIST);
-
- List<String> tablesList = new ArrayList<>();
- for (int child = 0; child < tablesListNode.getChildCount(); child++) {
- Tree tablePatternNode = tablesListNode.getChild(child);
- if (tablePatternNode.getType() == TOK_NULL) {
- LOG.error(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg());
- throw new SemanticException(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg());
- }
- tablesList.add(unescapeSQLString(tablePatternNode.getText()));
+ String tableList = unescapeSQLString(replTablesNode.getChild(listIdx).getText());
+ if (tableList == null || tableList.isEmpty()) {
+ throw new SemanticException(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN);
}
-
if (listIdx == 0) {
- LOG.info("{} ReplScope: Set Included Tables List: {}", replScopeType, tablesList);
- replScope.setIncludedTablePatterns(tablesList);
+ LOG.info("{} ReplScope: Set Included Tables List: {}", replScopeType, tableList);
+ replScope.setIncludedTablePatterns(tableList);
} else {
- LOG.info("{} ReplScope: Set Excluded Tables List: {}", replScopeType, tablesList);
- replScope.setExcludedTablePatterns(tablesList);
+ LOG.info("{} ReplScope: Set Excluded Tables List: {}", replScopeType, tableList);
+ replScope.setExcludedTablePatterns(tableList);
}
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
index 4c137eb..f275194 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
@@ -85,48 +85,26 @@ public class DumpMetaData {
}
String[] lineContents = line.split("\t");
- if (lineContents.length < 1) {
- return;
- }
-
replScope = new ReplScope();
-
- LOG.info("Read ReplScope: Set Db Name: {}.", lineContents[0]);
- replScope.setDbName(lineContents[0]);
-
- // Read/set include and exclude tables list.
- int idx = readReplScopeTablesList(lineContents, 1, true);
- readReplScopeTablesList(lineContents, idx, false);
- }
-
- private int readReplScopeTablesList(String[] lineContents, int startIdx, boolean includeList)
- throws IOException {
- // If the list doesn't exist, then return.
- if (startIdx >= lineContents.length) {
- return startIdx;
- }
-
- // Each tables list should start with "{" and ends with "}"
- if (!"{".equals(lineContents[startIdx])) {
- throw new IOException("Invalid repl tables list data in dump metadata file. Missing \"{\".");
- }
-
- List<String>tableNames = new ArrayList<>();
- for (int i = (startIdx + 1); i < lineContents.length; i++) {
- String value = lineContents[i];
- if ("}".equals(value)) {
- if (includeList) {
- LOG.info("Read ReplScope: Set Include Table Names: {}.", tableNames);
- replScope.setIncludedTablePatterns(tableNames);
- } else {
- LOG.info("Read ReplScope: Set Exclude Table Names: {}.", tableNames);
- replScope.setExcludedTablePatterns(tableNames);
- }
- return (i + 1);
+ for (int idx = 0; idx < lineContents.length; idx++) {
+ String value = lineContents[idx];
+ switch (idx) {
+ case 0:
+ LOG.info("Read ReplScope: Set Db Name: {}.", value);
+ replScope.setDbName(value);
+ break;
+ case 1:
+ LOG.info("Read ReplScope: Include table name list: {}.", value);
+ replScope.setIncludedTablePatterns(value);
+ break;
+ case 2:
+ LOG.info("Read ReplScope: Exclude table name list: {}.", value);
+ replScope.setExcludedTablePatterns(value);
+ break;
+ default:
+ throw new IOException("Invalid repl tables list data in dump metadata file");
}
- tableNames.add(value);
}
- throw new IOException("Invalid repl tables list data in dump metadata file. Missing \"}\".");
}
private void loadDumpFromFile() throws SemanticException {
@@ -205,17 +183,13 @@ public class DumpMetaData {
List<String> values = new ArrayList<>();
values.add(replScope.getDbName());
- List<String> includedTableNames = replScope.getIncludedTableNames();
- List<String> excludedTableNames = replScope.getExcludedTableNames();
+ String includedTableNames = replScope.getIncludedTableNames();
+ String excludedTableNames = replScope.getExcludedTableNames();
if (includedTableNames != null) {
- values.add("{");
- values.addAll(includedTableNames);
- values.add("}");
+ values.add(includedTableNames);
}
if (excludedTableNames != null) {
- values.add("{");
- values.addAll(excludedTableNames);
- values.add("}");
+ values.add(excludedTableNames);
}
LOG.info("Preparing ReplScope {} to dump.", values);
return values;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java
index 9db4e96..48b9883 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java
@@ -77,9 +77,7 @@ public class TestReplicationSemanticAnalyzer {
ASTNode child = (ASTNode) root.getChild(1);
assertEquals("TOK_REPL_TABLES", child.getText());
assertEquals(1, child.getChildCount());
- assertEquals("TOK_REPL_TABLES_LIST", child.getChild(0).getText());
- assertEquals(1, child.getChild(0).getChildCount());
- assertEquals("'test_table'", child.getChild(0).getChild(0).getText());
+ assertEquals("'test_table'", child.getChild(0).getText());
}
private static void assertDatabase(final int expectedNumberOfChildren, ASTNode root) {
@@ -110,14 +108,14 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseTableName() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table']");
+ ASTNode root = parse("repl dump testDb.'test_table'");
assertDatabase(2, root);
assertTableName(root);
}
@Test
public void parseFromEventId() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table'] from 100");
+ ASTNode root = parse("repl dump testDb.'test_table' from 100");
assertDatabase(3, root);
assertTableName(root);
assertFromEvent(1, root);
@@ -125,7 +123,7 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseToEventId() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200");
+ ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200");
assertDatabase(3, root);
assertTableName(root);
ASTNode fromClauseRootNode = assertFromEvent(3, root);
@@ -134,7 +132,7 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseLimit() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200 limit 10");
+ ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200 limit 10");
assertDatabase(3, root);
assertTableName(root);
ASTNode fromClauseRootNode = assertFromEvent(5, root);
@@ -162,7 +160,7 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseTableName() throws ParseException {
ASTNode root =
- parse("repl dump testDb.['test_table'] with ('key.1'='value.1','key.2'='value.2')");
+ parse("repl dump testDb.'test_table' with ('key.1'='value.1','key.2'='value.2')");
assertDatabase(3, root);
assertTableName(root);
assertWithClause(root, 2);
@@ -170,7 +168,7 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseFromEventId() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table'] from 100 "
+ ASTNode root = parse("repl dump testDb.'test_table' from 100 "
+ "with ('key.1'='value.1','key.2'='value.2')");
assertDatabase(4, root);
assertTableName(root);
@@ -180,7 +178,7 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseToEventId() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200 "
+ ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200 "
+ "with ('key.1'='value.1','key.2'='value.2')");
assertDatabase(4, root);
assertTableName(root);
@@ -191,7 +189,7 @@ public class TestReplicationSemanticAnalyzer {
@Test
public void parseLimit() throws ParseException {
- ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200 limit 10 "
+ ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200 limit 10 "
+ "with ('key.1'='value.1','key.2'='value.2')");
assertDatabase(4, root);
assertTableName(root);
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java
index 14b58a3..8b26f1c 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.common.repl;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
import java.util.regex.Pattern;
/**
@@ -31,10 +29,10 @@ public class ReplScope implements Serializable {
private Pattern dbNamePattern;
// Include and exclude table names/patterns exist only for REPL DUMP.
- private List<String> includedTableNames;
- private List<String> excludedTableNames;
- private List<Pattern> includedTableNamePatterns;
- private List<Pattern> excludedTableNamePatterns;
+ private String includedTableNames;
+ private String excludedTableNames;
+ private Pattern includedTableNamePattern;
+ private Pattern excludedTableNamePattern;
public ReplScope() {
}
@@ -53,27 +51,27 @@ public class ReplScope implements Serializable {
return dbName;
}
- public void setIncludedTablePatterns(List<String> includedTableNames) {
+ public void setIncludedTablePatterns(String includedTableNames) {
this.includedTableNames = includedTableNames;
- this.includedTableNamePatterns = compilePatterns(includedTableNames);
+ this.includedTableNamePattern = compilePattern(includedTableNames);
}
- public List<String> getIncludedTableNames() {
+ public String getIncludedTableNames() {
return includedTableNames;
}
- public void setExcludedTablePatterns(List<String> excludedTableNames) {
+ public void setExcludedTablePatterns(String excludedTableNames) {
this.excludedTableNames = excludedTableNames;
- this.excludedTableNamePatterns = compilePatterns(excludedTableNames);
+ this.excludedTableNamePattern = compilePattern(excludedTableNames);
}
- public List<String> getExcludedTableNames() {
+ public String getExcludedTableNames() {
return excludedTableNames;
}
public boolean includeAllTables() {
- return ((includedTableNamePatterns == null)
- && (excludedTableNamePatterns == null));
+ return ((includedTableNamePattern == null)
+ && (excludedTableNamePattern == null));
}
public boolean includedInReplScope(final String dbName, final String tableName) {
@@ -93,42 +91,29 @@ public class ReplScope implements Serializable {
return (inTableIncludedList(tableName) && !inTableExcludedList(tableName));
}
- private List<Pattern> compilePatterns(List<String> patterns) {
- if (patterns == null || patterns.isEmpty()) {
+ private Pattern compilePattern(String pattern) {
+ if (pattern == null) {
return null;
}
- List<Pattern> compiledPatterns = new ArrayList<>();
- for (String pattern : patterns) {
- // Convert the pattern to lower case because events/HMS will have table names in lower case.
- compiledPatterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE));
- }
- return compiledPatterns;
- }
- private boolean tableMatchAnyPattern(final String tableName, List<Pattern> tableNamePatterns) {
- assert(tableNamePatterns != null);
- for (Pattern tableNamePattern : tableNamePatterns) {
- if (tableNamePattern.matcher(tableName).matches()) {
- return true;
- }
- }
- return false;
+ // Convert the pattern to lower case because events/HMS will have table names in lower case.
+ return Pattern.compile(pattern, Pattern.CASE_INSENSITIVE);
}
private boolean inTableIncludedList(final String tableName) {
- if (includedTableNamePatterns == null) {
+ if (includedTableNamePattern == null) {
// If included list is empty, repl policy should be full db replication.
// So, all tables must be included.
return true;
}
- return tableMatchAnyPattern(tableName, includedTableNamePatterns);
+ return includedTableNamePattern.matcher(tableName).matches();
}
private boolean inTableExcludedList(final String tableName) {
- if (excludedTableNamePatterns == null) {
+ if (excludedTableNamePattern == null) {
// If excluded tables list is empty means, all tables in included list must be accepted.
return false;
}
- return tableMatchAnyPattern(tableName, excludedTableNamePatterns);
+ return excludedTableNamePattern.matcher(tableName).matches();
}
}