You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2022/01/02 16:42:26 UTC
[hbase] branch branch-2.5 updated: HBASE-26641 Split TestMasterFailoverWithProcedures (#3994)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new deec701 HBASE-26641 Split TestMasterFailoverWithProcedures (#3994)
deec701 is described below
commit deec70147514c2bc0f9aabc041b4d93c0a0b5fd9
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Jan 3 00:35:14 2022 +0800
HBASE-26641 Split TestMasterFailoverWithProcedures (#3994)
Signed-off-by: GeorryHuang <hu...@apache.org>
---
.../MasterFailoverWithProceduresTestBase.java | 87 ++++++
.../TestCreateTableWithMasterFailover.java | 74 +++++
.../TestDeleteTableWithMasterFailover.java | 76 +++++
.../TestDisableTableWithMasterFailover.java | 74 +++++
.../TestEnableTableWithMasterFailover.java | 74 +++++
.../TestMasterFailoverWithProcedures.java | 335 ---------------------
.../TestTruncateTableWithMasterFailover.java | 102 +++++++
7 files changed, 487 insertions(+), 335 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java
new file mode 100644
index 0000000..1793301
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.StartMiniClusterOption;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class MasterFailoverWithProceduresTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MasterFailoverWithProceduresTestBase.class);
+
+ protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ StartMiniClusterOption option = StartMiniClusterOption.builder().numMasters(2).build();
+ UTIL.startMiniCluster(option);
+
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, false);
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, false);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ // ==========================================================================
+ // Helpers
+ // ==========================================================================
+ protected static ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ }
+
+ protected static Path getRootDir() {
+ return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+ }
+
+ protected static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil,
+ final long procId, final int lastStepBeforeFailover) throws Exception {
+ ProcedureExecutor<MasterProcedureEnv> procExec =
+ testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+
+ final Procedure<?> proc = procExec.getProcedure(procId);
+ for (int i = 0; i < lastStepBeforeFailover; ++i) {
+ LOG.info("Restart " + i + " exec state: " + proc);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
+ MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ }
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
+
+ LOG.info("Trigger master failover");
+ MasterProcedureTestingUtility.masterFailover(testUtil);
+
+ procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java
new file mode 100644
index 0000000..3d4740a
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CreateTableState;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestCreateTableWithMasterFailover extends MasterFailoverWithProceduresTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestCreateTableWithMasterFailover.class);
+
+ // ==========================================================================
+ // Test Create Table
+ // ==========================================================================
+ @Test
+ public void testCreateWithFailover() throws Exception {
+ // TODO: Should we try every step? (master failover takes long time)
+ // It is already covered by TestCreateTableProcedure
+ // but without the master restart, only the executor/store is restarted.
+ // Without Master restart we may not find bug in the procedure code
+ // like missing "wait" for resources to be available (e.g. RS)
+ testCreateWithFailoverAtStep(CreateTableState.CREATE_TABLE_ASSIGN_REGIONS.ordinal());
+ }
+
+ private void testCreateWithFailoverAtStep(final int step) throws Exception {
+ final TableName tableName = TableName.valueOf("testCreateWithFailoverAtStep" + step);
+
+ // create the table
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);
+
+ // Start the Create procedure && kill the executor
+ byte[][] splitKeys = null;
+ TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
+ RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
+ long procId =
+ procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
+ testRecoveryAndDoubleExecution(UTIL, procId, step);
+
+ MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
+ tableName, regions, "f1", "f2");
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java
new file mode 100644
index 0000000..66990da
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestDeleteTableWithMasterFailover extends MasterFailoverWithProceduresTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDeleteTableWithMasterFailover.class);
+
+ // ==========================================================================
+ // Test Delete Table
+ // ==========================================================================
+ @Test
+ public void testDeleteWithFailover() throws Exception {
+ // TODO: Should we try every step? (master failover takes long time)
+ // It is already covered by TestDeleteTableProcedure
+ // but without the master restart, only the executor/store is restarted.
+ // Without Master restart we may not find bug in the procedure code
+ // like missing "wait" for resources to be available (e.g. RS)
+ testDeleteWithFailoverAtStep(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS.ordinal());
+ }
+
+ private void testDeleteWithFailoverAtStep(final int step) throws Exception {
+ final TableName tableName = TableName.valueOf("testDeleteWithFailoverAtStep" + step);
+
+ // create the table
+ byte[][] splitKeys = null;
+ RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
+ tableName, splitKeys, "f1", "f2");
+ MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
+ tableName, regions, "f1", "f2");
+ UTIL.getAdmin().disableTable(tableName);
+
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);
+
+ // Start the Delete procedure && kill the executor
+ long procId =
+ procExec.submitProcedure(new DeleteTableProcedure(procExec.getEnvironment(), tableName));
+ testRecoveryAndDoubleExecution(UTIL, procId, step);
+
+ MasterProcedureTestingUtility.validateTableDeletion(UTIL.getHBaseCluster().getMaster(),
+ tableName);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java
new file mode 100644
index 0000000..4a54b1c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DisableTableState;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestDisableTableWithMasterFailover extends MasterFailoverWithProceduresTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestDisableTableWithMasterFailover.class);
+
+ // ==========================================================================
+ // Test Disable Table
+ // ==========================================================================
+ @Test
+ public void testDisableTableWithFailover() throws Exception {
+ // TODO: Should we try every step? (master failover takes long time)
+ // It is already covered by TestDisableTableProcedure
+ // but without the master restart, only the executor/store is restarted.
+ // Without Master restart we may not find bug in the procedure code
+ // like missing "wait" for resources to be available (e.g. RS)
+ testDisableTableWithFailoverAtStep(
+ DisableTableState.DISABLE_TABLE_MARK_REGIONS_OFFLINE.ordinal());
+ }
+
+ private void testDisableTableWithFailoverAtStep(final int step) throws Exception {
+ final TableName tableName = TableName.valueOf("testDisableTableWithFailoverAtStep" + step);
+
+ // create the table
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), tableName, splitKeys,
+ "f1", "f2");
+
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Start the Delete procedure && kill the executor
+ long procId = procExec
+ .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), tableName, false));
+ testRecoveryAndDoubleExecution(UTIL, procId, step);
+
+ MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(),
+ tableName);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java
new file mode 100644
index 0000000..0e47cf4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestEnableTableWithMasterFailover extends MasterFailoverWithProceduresTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestEnableTableWithMasterFailover.class);
+
+ // ==========================================================================
+ // Test Enable Table
+ // ==========================================================================
+ @Test
+ public void testEnableTableWithFailover() throws Exception {
+ // TODO: Should we try every step? (master failover takes long time)
+ // It is already covered by TestEnableTableProcedure
+ // but without the master restart, only the executor/store is restarted.
+ // Without Master restart we may not find bug in the procedure code
+ // like missing "wait" for resources to be available (e.g. RS)
+ testEnableTableWithFailoverAtStep(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE.ordinal());
+ }
+
+ private void testEnableTableWithFailoverAtStep(final int step) throws Exception {
+ final TableName tableName = TableName.valueOf("testEnableTableWithFailoverAtStep" + step);
+
+ // create the table
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), tableName, splitKeys,
+ "f1", "f2");
+ UTIL.getAdmin().disableTable(tableName);
+
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Start the Delete procedure && kill the executor
+ long procId =
+ procExec.submitProcedure(new EnableTableProcedure(procExec.getEnvironment(), tableName));
+ testRecoveryAndDoubleExecution(UTIL, procId, step);
+
+ MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(),
+ tableName);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
deleted file mode 100644
index 1253710..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.procedure;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.StartMiniClusterOption;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.procedure2.Procedure;
-import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
-import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.CommonFSUtils;
-import org.apache.hadoop.hbase.util.ModifyRegionUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CreateTableState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DisableTableState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateTableState;
-
-@Category({MasterTests.class, LargeTests.class})
-public class TestMasterFailoverWithProcedures {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestMasterFailoverWithProcedures.class);
-
- private static final Logger LOG = LoggerFactory.getLogger(TestMasterFailoverWithProcedures.class);
-
- protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- private static void setupConf(Configuration conf) {
- // don't waste time retrying with the roll, the test is already slow enough.
- conf.setInt(WALProcedureStore.MAX_RETRIES_BEFORE_ROLL_CONF_KEY, 1);
- conf.setInt(WALProcedureStore.WAIT_BEFORE_ROLL_CONF_KEY, 0);
- conf.setInt(WALProcedureStore.ROLL_RETRIES_CONF_KEY, 1);
- conf.setInt(WALProcedureStore.MAX_SYNC_FAILURE_ROLL_CONF_KEY, 1);
- conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
- }
-
- @Before
- public void setup() throws Exception {
- setupConf(UTIL.getConfiguration());
- // Set master number and use default values for other options.
- StartMiniClusterOption option = StartMiniClusterOption.builder().numMasters(2).build();
- UTIL.startMiniCluster(option);
-
- final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
- ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, false);
- ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, false);
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- LOG.warn("failure shutting down cluster", e);
- }
- }
-
- // ==========================================================================
- // Test Create Table
- // ==========================================================================
- @Test
- public void testCreateWithFailover() throws Exception {
- // TODO: Should we try every step? (master failover takes long time)
- // It is already covered by TestCreateTableProcedure
- // but without the master restart, only the executor/store is restarted.
- // Without Master restart we may not find bug in the procedure code
- // like missing "wait" for resources to be available (e.g. RS)
- testCreateWithFailoverAtStep(CreateTableState.CREATE_TABLE_ASSIGN_REGIONS.ordinal());
- }
-
- private void testCreateWithFailoverAtStep(final int step) throws Exception {
- final TableName tableName = TableName.valueOf("testCreateWithFailoverAtStep" + step);
-
- // create the table
- ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
- ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
- ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);
-
- // Start the Create procedure && kill the executor
- byte[][] splitKeys = null;
- TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2");
- RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
- long procId = procExec.submitProcedure(
- new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
- testRecoveryAndDoubleExecution(UTIL, procId, step);
-
- MasterProcedureTestingUtility.validateTableCreation(
- UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
- }
-
- // ==========================================================================
- // Test Delete Table
- // ==========================================================================
- @Test
- public void testDeleteWithFailover() throws Exception {
- // TODO: Should we try every step? (master failover takes long time)
- // It is already covered by TestDeleteTableProcedure
- // but without the master restart, only the executor/store is restarted.
- // Without Master restart we may not find bug in the procedure code
- // like missing "wait" for resources to be available (e.g. RS)
- testDeleteWithFailoverAtStep(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS.ordinal());
- }
-
- private void testDeleteWithFailoverAtStep(final int step) throws Exception {
- final TableName tableName = TableName.valueOf("testDeleteWithFailoverAtStep" + step);
-
- // create the table
- byte[][] splitKeys = null;
- RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
- getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
- Path tableDir = CommonFSUtils.getTableDir(getRootDir(), tableName);
- MasterProcedureTestingUtility.validateTableCreation(
- UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
- UTIL.getAdmin().disableTable(tableName);
-
- ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
- ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true);
- ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true);
-
- // Start the Delete procedure && kill the executor
- long procId = procExec.submitProcedure(
- new DeleteTableProcedure(procExec.getEnvironment(), tableName));
- testRecoveryAndDoubleExecution(UTIL, procId, step);
-
- MasterProcedureTestingUtility.validateTableDeletion(
- UTIL.getHBaseCluster().getMaster(), tableName);
- }
-
- // ==========================================================================
- // Test Truncate Table
- // ==========================================================================
- @Test
- public void testTruncateWithFailover() throws Exception {
- // TODO: Should we try every step? (master failover takes long time)
- // It is already covered by TestTruncateTableProcedure
- // but without the master restart, only the executor/store is restarted.
- // Without Master restart we may not find bug in the procedure code
- // like missing "wait" for resources to be available (e.g. RS)
- testTruncateWithFailoverAtStep(true, TruncateTableState.TRUNCATE_TABLE_ADD_TO_META.ordinal());
- }
-
- private void testTruncateWithFailoverAtStep(final boolean preserveSplits, final int step)
- throws Exception {
- final TableName tableName = TableName.valueOf("testTruncateWithFailoverAtStep" + step);
-
- // create the table
- final String[] families = new String[] { "f1", "f2" };
- final byte[][] splitKeys = new byte[][] {
- Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
- };
- RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
- getMasterProcedureExecutor(), tableName, splitKeys, families);
- // load and verify that there are rows in the table
- MasterProcedureTestingUtility.loadData(
- UTIL.getConnection(), tableName, 100, splitKeys, families);
- assertEquals(100, UTIL.countRows(tableName));
- // disable the table
- UTIL.getAdmin().disableTable(tableName);
-
- ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
- ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
-
- // Start the Truncate procedure && kill the executor
- long procId = procExec.submitProcedure(
- new TruncateTableProcedure(procExec.getEnvironment(), tableName, preserveSplits));
- testRecoveryAndDoubleExecution(UTIL, procId, step);
-
- ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
- UTIL.waitUntilAllRegionsAssigned(tableName);
-
- // validate the table regions and layout
- regions = UTIL.getAdmin().getTableRegions(tableName).toArray(new RegionInfo[0]);
- if (preserveSplits) {
- assertEquals(1 + splitKeys.length, regions.length);
- } else {
- assertEquals(1, regions.length);
- }
- MasterProcedureTestingUtility.validateTableCreation(
- UTIL.getHBaseCluster().getMaster(), tableName, regions, families);
-
- // verify that there are no rows in the table
- assertEquals(0, UTIL.countRows(tableName));
-
- // verify that the table is read/writable
- MasterProcedureTestingUtility.loadData(
- UTIL.getConnection(), tableName, 50, splitKeys, families);
- assertEquals(50, UTIL.countRows(tableName));
- }
-
- // ==========================================================================
- // Test Disable Table
- // ==========================================================================
- @Test
- public void testDisableTableWithFailover() throws Exception {
- // TODO: Should we try every step? (master failover takes long time)
- // It is already covered by TestDisableTableProcedure
- // but without the master restart, only the executor/store is restarted.
- // Without Master restart we may not find bug in the procedure code
- // like missing "wait" for resources to be available (e.g. RS)
- testDisableTableWithFailoverAtStep(
- DisableTableState.DISABLE_TABLE_MARK_REGIONS_OFFLINE.ordinal());
- }
-
- private void testDisableTableWithFailoverAtStep(final int step) throws Exception {
- final TableName tableName = TableName.valueOf("testDisableTableWithFailoverAtStep" + step);
-
- // create the table
- final byte[][] splitKeys = new byte[][] {
- Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
- };
- MasterProcedureTestingUtility.createTable(
- getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
-
- ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
- ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
-
- // Start the Delete procedure && kill the executor
- long procId = procExec.submitProcedure(
- new DisableTableProcedure(procExec.getEnvironment(), tableName, false));
- testRecoveryAndDoubleExecution(UTIL, procId, step);
-
- MasterProcedureTestingUtility.validateTableIsDisabled(
- UTIL.getHBaseCluster().getMaster(), tableName);
- }
-
- // ==========================================================================
- // Test Enable Table
- // ==========================================================================
- @Test
- public void testEnableTableWithFailover() throws Exception {
- // TODO: Should we try every step? (master failover takes long time)
- // It is already covered by TestEnableTableProcedure
- // but without the master restart, only the executor/store is restarted.
- // Without Master restart we may not find bug in the procedure code
- // like missing "wait" for resources to be available (e.g. RS)
- testEnableTableWithFailoverAtStep(
- EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE.ordinal());
- }
-
- private void testEnableTableWithFailoverAtStep(final int step) throws Exception {
- final TableName tableName = TableName.valueOf("testEnableTableWithFailoverAtStep" + step);
-
- // create the table
- final byte[][] splitKeys = new byte[][] {
- Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
- };
- MasterProcedureTestingUtility.createTable(
- getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
- UTIL.getAdmin().disableTable(tableName);
-
- ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
- ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
-
- // Start the Delete procedure && kill the executor
- long procId = procExec.submitProcedure(
- new EnableTableProcedure(procExec.getEnvironment(), tableName));
- testRecoveryAndDoubleExecution(UTIL, procId, step);
-
- MasterProcedureTestingUtility.validateTableIsEnabled(
- UTIL.getHBaseCluster().getMaster(), tableName);
- }
-
- // ==========================================================================
- // Test Helpers
- // ==========================================================================
- public static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil,
- final long procId, final int lastStepBeforeFailover) throws Exception {
- ProcedureExecutor<MasterProcedureEnv> procExec =
- testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
- ProcedureTestingUtility.waitProcedure(procExec, procId);
-
- final Procedure proc = procExec.getProcedure(procId);
- for (int i = 0; i < lastStepBeforeFailover; ++i) {
- LOG.info("Restart "+ i +" exec state: " + proc);
- ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
- MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);
- ProcedureTestingUtility.waitProcedure(procExec, procId);
- }
- ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId);
-
- LOG.info("Trigger master failover");
- MasterProcedureTestingUtility.masterFailover(testUtil);
-
- procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor();
- ProcedureTestingUtility.waitProcedure(procExec, procId);
- ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
- }
-
- // ==========================================================================
- // Helpers
- // ==========================================================================
- private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
- return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
- }
-
- private Path getRootDir() {
- return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
- }
-}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java
new file mode 100644
index 0000000..4ee39c9
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateTableState;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestTruncateTableWithMasterFailover extends MasterFailoverWithProceduresTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestTruncateTableWithMasterFailover.class);
+
+ // ==========================================================================
+ // Test Truncate Table
+ // ==========================================================================
+ @Test
+ public void testTruncateWithFailover() throws Exception {
+ // TODO: Should we try every step? (master failover takes long time)
+ // It is already covered by TestTruncateTableProcedure
+ // but without the master restart, only the executor/store is restarted.
+ // Without Master restart we may not find bug in the procedure code
+ // like missing "wait" for resources to be available (e.g. RS)
+ testTruncateWithFailoverAtStep(true, TruncateTableState.TRUNCATE_TABLE_ADD_TO_META.ordinal());
+ }
+
+ private void testTruncateWithFailoverAtStep(final boolean preserveSplits, final int step)
+ throws Exception {
+ final TableName tableName = TableName.valueOf("testTruncateWithFailoverAtStep" + step);
+
+ // create the table
+ final String[] families = new String[] { "f1", "f2" };
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
+ tableName, splitKeys, families);
+ // load and verify that there are rows in the table
+ MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, splitKeys,
+ families);
+ assertEquals(100, UTIL.countRows(tableName));
+ // disable the table
+ UTIL.getAdmin().disableTable(tableName);
+
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Start the Truncate procedure && kill the executor
+ long procId = procExec.submitProcedure(
+ new TruncateTableProcedure(procExec.getEnvironment(), tableName, preserveSplits));
+ testRecoveryAndDoubleExecution(UTIL, procId, step);
+
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
+ UTIL.waitUntilAllRegionsAssigned(tableName);
+
+ // validate the table regions and layout
+ regions = UTIL.getAdmin().getRegions(tableName).toArray(new RegionInfo[0]);
+ if (preserveSplits) {
+ assertEquals(1 + splitKeys.length, regions.length);
+ } else {
+ assertEquals(1, regions.length);
+ }
+ MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
+ tableName, regions, families);
+
+ // verify that there are no rows in the table
+ assertEquals(0, UTIL.countRows(tableName));
+
+ // verify that the table is read/writable
+ MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 50, splitKeys,
+ families);
+ assertEquals(50, UTIL.countRows(tableName));
+ }
+}