You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/25 20:49:44 UTC
[05/31] hive git commit: HIVE-16706 : Bootstrap REPL DUMP shouldn't
fail when a partition is dropped/renamed when dump in progress (Sankar
Hariappan via Thejas Nair)
HIVE-16706 : Bootstrap REPL DUMP shouldn't fail when a partition is dropped/renamed when dump in progress (Sankar Hariappan via Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c1b354e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c1b354e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c1b354e
Branch: refs/heads/hive-14535
Commit: 3c1b354eadb2033681efe3c06a99920ba4912a13
Parents: cc78d03
Author: Sankar Hariappan <ma...@gmail.com>
Authored: Wed May 24 23:26:23 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Wed May 24 23:26:34 2017 -0700
----------------------------------------------------------------------
.../hive/ql/parse/TestReplicationScenarios.java | 171 ++++++++++++++-----
.../InjectableBehaviourObjectStore.java | 20 +++
2 files changed, 144 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1b354e/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 0aee012..21f09ae 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -386,7 +386,6 @@ public class TestReplicationScenarios {
run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
-
advanceDumpDir();
BehaviourInjection<Table,Table> ptnedTableNuller = new BehaviourInjection<Table,Table>(){
@@ -414,12 +413,14 @@ public class TestReplicationScenarios {
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- // The ptned table should miss in target as the table was marked cisrtually as dropped
+ // The ptned table should miss in target as the table was marked virtually as dropped
verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
+ verifyIfTableNotExist(dbName + "_dupe", "ptned");
// Verify if Drop table on a non-existing table is idempotent
run("DROP TABLE " + dbName + ".ptned");
+ verifyIfTableNotExist(dbName, "ptned");
advanceDumpDir();
run("REPL DUMP " + dbName + " FROM " + replDumpId);
@@ -429,10 +430,82 @@ public class TestReplicationScenarios {
assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true));
verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
+ verifyIfTableNotExist(dbName + "_dupe", "ptned");
verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
}
@Test
+ public void testBootstrapWithConcurrentDropPartition() throws IOException {
+ String name = testName.getMethodName();
+ String dbName = createDB(name);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+
+ String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
+ String[] ptn_data_2 = new String[]{ "fifteen", "sixteen", "seventeen"};
+ String[] empty = new String[]{};
+
+ String ptn_locn_1 = new Path(TEST_PATH, name + "_ptn1").toUri().getPath();
+ String ptn_locn_2 = new Path(TEST_PATH, name + "_ptn2").toUri().getPath();
+
+ createTestDataFile(ptn_locn_1, ptn_data_1);
+ createTestDataFile(ptn_locn_2, ptn_data_2);
+
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
+
+ advanceDumpDir();
+
+ BehaviourInjection<List<String>, List<String>> listPartitionNamesNuller
+ = new BehaviourInjection<List<String>, List<String>>(){
+ @Nullable
+ @Override
+ public List<String> apply(@Nullable List<String> partitions) {
+ injectionPathCalled = true;
+ return new ArrayList<String>();
+ }
+ };
+ InjectableBehaviourObjectStore.setListPartitionNamesBehaviour(listPartitionNamesNuller);
+
+ // None of the partitions will be dumped as the partitions list was empty
+ run("REPL DUMP " + dbName);
+ listPartitionNamesNuller.assertInjectionsPerformed(true, false);
+ InjectableBehaviourObjectStore.resetListPartitionNamesBehaviour(); // reset the behaviour
+
+ String replDumpLocn = getResult(0, 0);
+ String replDumpId = getResult(0, 1, true);
+ LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+
+ // All partitions should miss in target as it was marked virtually as dropped
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")));
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")));
+
+ // Verify if drop partition on a non-existing partition is idempotent and just a noop.
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=1)");
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=2)");
+ verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("1")));
+ verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("2")));
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", empty);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", empty);
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId);
+ String postDropReplDumpLocn = getResult(0,0);
+ String postDropReplDumpId = getResult(0,1,true);
+ LOG.info("Dumped to {} with id {}->{}", postDropReplDumpLocn, replDumpId, postDropReplDumpId);
+ assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true));
+
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")));
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")));
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
+ }
+
+ @Test
public void testIncrementalAdds() throws IOException {
String name = testName.getMethodName();
String dbName = createDB(name);
@@ -599,30 +672,14 @@ public class TestReplicationScenarios {
// not existing, and thus, throwing a NoSuchObjectException, or returning nulls
// or select * returning empty, depending on what we're testing.
- Exception e = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe", "unptned");
- assertNull(tbl);
- } catch (TException te) {
- e = te;
- }
- assertNotNull(e);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "unptned");
verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", empty);
verifyRun("SELECT a from " + dbName + "_dupe.ptned", ptn_data_1);
verifyRun("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=1", empty);
verifyRun("SELECT a from " + dbName + "_dupe.ptned3", ptn_data_2);
- Exception e2 = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName+"_dupe","ptned2");
- assertNull(tbl);
- } catch (TException te) {
- e2 = te;
- }
- assertNotNull(e2);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "ptned2");
}
@Test
@@ -732,15 +789,7 @@ public class TestReplicationScenarios {
run("SELECT a from " + dbName + "_dupe.ptned");
verifyResults(ptn_data_1);
- Exception e2 = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName+"_dupe","ptned2");
- assertNull(tbl);
- } catch (TException te) {
- e2 = te;
- }
- assertNotNull(e2);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName +"_dupe", "ptned2");
run("SELECT a from " + dbName + "_dupe.unptned_copy");
verifyResults(unptn_data);
@@ -874,15 +923,7 @@ public class TestReplicationScenarios {
// Replication done, we now do the following verifications:
// verify that unpartitioned table rename succeeded.
- Exception e = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe" , "unptned");
- assertNull(tbl);
- } catch (TException te) {
- e = te;
- }
- assertNotNull(e);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "unptned");
verifyRun("SELECT * from " + dbName + "_dupe.unptned_rn", unptn_data);
// verify that partition rename succeded.
@@ -898,15 +939,7 @@ public class TestReplicationScenarios {
verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=22", ptn_data_2);
// verify that ptned table rename succeded.
- Exception e2 = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe" , "ptned2");
- assertNull(tbl);
- } catch (TException te) {
- e2 = te;
- }
- assertNotNull(e2);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "ptned2");
verifyRun("SELECT a from " + dbName + "_dupe.ptned2_rn WHERE b=2", ptn_data_2);
// verify that ptned table property set worked
@@ -1970,6 +2003,50 @@ public class TestReplicationScenarios {
}
}
+ private void verifyIfTableNotExist(String dbName, String tableName){
+ Exception e = null;
+ try {
+ Table tbl = metaStoreClient.getTable(dbName, tableName);
+ assertNull(tbl);
+ } catch (TException te) {
+ e = te;
+ }
+ assertNotNull(e);
+ assertEquals(NoSuchObjectException.class, e.getClass());
+ }
+
+ private void verifyIfTableExist(String dbName, String tableName){
+ Exception e = null;
+ try {
+ Table tbl = metaStoreClient.getTable(dbName, tableName);
+ assertNotNull(tbl);
+ } catch (TException te) {
+ assert(false);
+ }
+ }
+
+ private void verifyIfPartitionNotExist(String dbName, String tableName, List<String> partValues){
+ Exception e = null;
+ try {
+ Partition ptn = metaStoreClient.getPartition(dbName, tableName, partValues);
+ assertNull(ptn);
+ } catch (TException te) {
+ e = te;
+ }
+ assertNotNull(e);
+ assertEquals(NoSuchObjectException.class, e.getClass());
+ }
+
+ private void verifyIfPartitionExist(String dbName, String tableName, List<String> partValues){
+ Exception e = null;
+ try {
+ Partition ptn = metaStoreClient.getPartition(dbName, tableName, partValues);
+ assertNotNull(ptn);
+ } catch (TException te) {
+ assert(false);
+ }
+ }
+
private void verifySetup(String cmd, String[] data) throws IOException {
if (VERIFY_SETUP_STEPS){
run(cmd);
http://git-wip-us.apache.org/repos/asf/hive/blob/3c1b354e/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
index a832c78..ed6d4be 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import java.util.List;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -50,6 +51,8 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
private static com.google.common.base.Function<Table,Table> getTableModifier =
com.google.common.base.Functions.identity();
+ private static com.google.common.base.Function<List<String>, List<String>> listPartitionNamesModifier =
+ com.google.common.base.Functions.identity();
public static void setGetTableBehaviour(com.google.common.base.Function<Table,Table> modifier){
getTableModifier = (modifier == null)? com.google.common.base.Functions.identity() : modifier;
@@ -63,8 +66,25 @@ public class InjectableBehaviourObjectStore extends ObjectStore {
return getTableModifier;
}
+ public static void setListPartitionNamesBehaviour(com.google.common.base.Function<List<String>, List<String>> modifier){
+ listPartitionNamesModifier = (modifier == null)? com.google.common.base.Functions.identity() : modifier;
+ }
+
+ public static void resetListPartitionNamesBehaviour(){
+ setListPartitionNamesBehaviour(null);
+ }
+
+ public static com.google.common.base.Function<List<String>, List<String>> getListPartitionNamesBehaviour() {
+ return listPartitionNamesModifier;
+ }
+
@Override
public Table getTable(String dbName, String tableName) throws MetaException {
return getTableModifier.apply(super.getTable(dbName, tableName));
}
+
+ @Override
+ public List<String> listPartitionNames(String dbName, String tableName, short max) throws MetaException {
+ return listPartitionNamesModifier.apply(super.listPartitionNames(dbName, tableName, max));
+ }
}