You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2020/06/27 03:17:19 UTC
[hbase] branch branch-2 updated: HBASE-24635 Split
TestMetaWithReplicas (#1980)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 7c78356 HBASE-24635 Split TestMetaWithReplicas (#1980)
7c78356 is described below
commit 7c78356218410d709335b603427a677ec5413b0f
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Jun 27 10:36:07 2020 +0800
HBASE-24635 Split TestMetaWithReplicas (#1980)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.../hadoop/hbase/IntegrationTestMetaReplicas.java | 5 +-
.../hbase/client/MetaWithReplicasTestBase.java | 123 ++++++
.../client/TestFailedMetaReplicaAssigment.java | 135 +++++++
.../client/TestMetaReplicasAddressChange.java | 92 +++++
.../hadoop/hbase/client/TestMetaWithReplicas.java | 427 ---------------------
.../hbase/client/TestMetaWithReplicasBasic.java | 91 +++++
.../TestMetaWithReplicasShutdownHandling.java | 164 ++++++++
.../client/TestShutdownOfMetaReplicaHolder.java | 67 ++++
8 files changed, 674 insertions(+), 430 deletions(-)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java
index f14b9a5..22c9b40 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMetaReplicas.java
@@ -18,9 +18,8 @@
package org.apache.hadoop.hbase;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.TestMetaWithReplicas;
+import org.apache.hadoop.hbase.client.TestMetaWithReplicasShutdownHandling;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
import org.apache.hadoop.hbase.testclassification.IntegrationTests;
@@ -98,7 +97,7 @@ public class IntegrationTestMetaReplicas {
// server holding the primary meta replica. Then it does a put/get into/from
// the test table. The put/get operations would use the replicas to locate the
// location of the test table's region
- TestMetaWithReplicas.shutdownMetaAndDoValidations(util);
+ TestMetaWithReplicasShutdownHandling.shutdownMetaAndDoValidations(util);
}
public static void main(String[] args) throws Exception {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java
new file mode 100644
index 0000000..78e3e54
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/MetaWithReplicasTestBase.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.StartMiniClusterOption;
+import org.apache.hadoop.hbase.TableNameTestRule;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
+import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
+import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for testing the scenarios where replicas are enabled for the meta table.
+ */
+public class MetaWithReplicasTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetaWithReplicasTestBase.class);
+
+ protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ protected static final int REGIONSERVERS_COUNT = 3;
+
+ @Rule
+ public TableNameTestRule name = new TableNameTestRule();
+
+ protected static void startCluster() throws Exception {
+ TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000);
+ TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
+ TEST_UTIL.getConfiguration()
+ .setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
+ StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1)
+ .numMasters(1).numRegionServers(REGIONSERVERS_COUNT).build();
+ TEST_UTIL.startMiniCluster(option);
+ AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
+ Set<ServerName> sns = new HashSet<ServerName>();
+ ServerName hbaseMetaServerName =
+ MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher());
+ LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName);
+ sns.add(hbaseMetaServerName);
+ for (int replicaId = 1; replicaId < 3; replicaId++) {
+ RegionInfo h = RegionReplicaUtil
+ .getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
+ AssignmentTestingUtil.waitForAssignment(am, h);
+ ServerName sn = am.getRegionStates().getRegionServerOfRegion(h);
+ assertNotNull(sn);
+ LOG.info("HBASE:META DEPLOY: " + h.getRegionNameAsString() + " on " + sn);
+ sns.add(sn);
+ }
+ // Fun. All meta region replicas have ended up on the one server. This will cause this test
+ // to fail ... sometimes.
+ if (sns.size() == 1) {
+ int count = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
+ assertTrue("count=" + count, count == REGIONSERVERS_COUNT);
+ LOG.warn("All hbase:meta replicas are on the one server; moving hbase:meta: " + sns);
+ int metaServerIndex = TEST_UTIL.getHBaseCluster().getServerWithMeta();
+ int newServerIndex = metaServerIndex;
+ while (newServerIndex == metaServerIndex) {
+ newServerIndex = (newServerIndex + 1) % REGIONSERVERS_COUNT;
+ }
+ assertNotEquals(metaServerIndex, newServerIndex);
+ ServerName destinationServerName =
+ TEST_UTIL.getHBaseCluster().getRegionServer(newServerIndex).getServerName();
+ ServerName metaServerName =
+ TEST_UTIL.getHBaseCluster().getRegionServer(metaServerIndex).getServerName();
+ assertNotEquals(destinationServerName, metaServerName);
+ TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
+ destinationServerName);
+ }
+ // Disable the balancer
+ LoadBalancerTracker l =
+ new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(), new Abortable() {
+ AtomicBoolean aborted = new AtomicBoolean(false);
+
+ @Override
+ public boolean isAborted() {
+ return aborted.get();
+ }
+
+ @Override
+ public void abort(String why, Throwable e) {
+ aborted.set(true);
+ }
+ });
+ l.setBalancerOn(false);
+ LOG.debug("All meta replicas assigned");
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java
new file mode 100644
index 0000000..0c26d79
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFailedMetaReplicaAssigment.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.concurrent.Future;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.StartMiniClusterOption;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
+import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
+import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestFailedMetaReplicaAssigment {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestFailedMetaReplicaAssigment.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // using our rigged master, to force a failed meta replica assignment
+ Configuration conf = TEST_UTIL.getConfiguration();
+ conf.setInt(HConstants.META_REPLICAS_NUM, 3);
+ StartMiniClusterOption option = StartMiniClusterOption.builder().numAlwaysStandByMasters(1)
+ .numMasters(1).numRegionServers(1).masterClass(BrokenMetaReplicaMaster.class).build();
+ TEST_UTIL.startMiniCluster(option);
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testFailedReplicaAssignment() throws InterruptedException {
+ HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+ // waiting for master to come up
+ TEST_UTIL.waitFor(30000, () -> master.isInitialized());
+
+ AssignmentManager am = master.getAssignmentManager();
+ // showing one of the replicas got assigned
+ RegionInfo metaReplicaHri =
+ RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 1);
+ // we use assignAsync so we need to wait a bit
+ TEST_UTIL.waitFor(30000, () -> {
+ RegionStateNode metaReplicaRegionNode =
+ am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri);
+ return metaReplicaRegionNode.getRegionLocation() != null;
+ });
+ // showing one of the replicas failed to be assigned
+ RegionInfo metaReplicaHri2 =
+ RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, 2);
+ RegionStateNode metaReplicaRegionNode2 =
+ am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2);
+ // wait for several seconds to make sure that it is not assigned
+ for (int i = 0; i < 3; i++) {
+ Thread.sleep(2000);
+ assertNull(metaReplicaRegionNode2.getRegionLocation());
+ }
+
+ // showing master is active and running
+ assertFalse(master.isStopping());
+ assertFalse(master.isStopped());
+ assertTrue(master.isActiveMaster());
+ }
+
+ public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure {
+ protected BrokenTransitRegionStateProcedure() {
+ super(null, null, null, false, TransitionType.ASSIGN);
+ }
+ }
+
+ public static class BrokenMetaReplicaMaster extends HMaster {
+ public BrokenMetaReplicaMaster(final Configuration conf) throws IOException {
+ super(conf);
+ }
+
+ @Override
+ public AssignmentManager createAssignmentManager(MasterServices master) {
+ return new BrokenMasterMetaAssignmentManager(master);
+ }
+ }
+
+ public static class BrokenMasterMetaAssignmentManager extends AssignmentManager {
+ MasterServices master;
+
+ public BrokenMasterMetaAssignmentManager(final MasterServices master) {
+ super(master);
+ this.master = master;
+ }
+
+ public Future<byte[]> assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException {
+ RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(regionInfo);
+ if (regionNode.getRegionInfo().getReplicaId() == 2) {
+ regionNode.setProcedure(new BrokenTransitRegionStateProcedure());
+ }
+ return super.assignAsync(regionInfo, sn);
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java
new file mode 100644
index 0000000..05e91ac
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaReplicasAddressChange.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collection;
+import java.util.EnumSet;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterMetrics.Option;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestMetaReplicasAddressChange extends MetaWithReplicasTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMetaReplicasAddressChange.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestMetaReplicasAddressChange.class);
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ startCluster();
+ }
+
+ @Test
+ public void testMetaAddressChange() throws Exception {
+ // checks that even when the meta's location changes, the various
+ // caches update themselves. Uses the master operations to test
+ // this
+ Configuration conf = TEST_UTIL.getConfiguration();
+ ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
+ String baseZNode =
+ conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+ String primaryMetaZnode =
+ ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
+ // check that the data in the znode is parseable (this would also mean the znode exists)
+ byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
+ ServerName currentServer = ProtobufUtil.parseServerNameFrom(data);
+ Collection<ServerName> liveServers = TEST_UTIL.getAdmin()
+ .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet();
+ ServerName moveToServer =
+ liveServers.stream().filter(s -> !currentServer.equals(s)).findAny().get();
+ final TableName tableName = name.getTableName();
+ TEST_UTIL.createTable(tableName, "f");
+ assertTrue(TEST_UTIL.getAdmin().tableExists(tableName));
+ TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
+ moveToServer);
+ assertNotEquals(currentServer, moveToServer);
+ LOG.debug("CurrentServer={}, moveToServer={}", currentServer, moveToServer);
+ TEST_UTIL.waitFor(60000, () -> {
+ byte[] bytes = ZKUtil.getData(zkw, primaryMetaZnode);
+ ServerName actualServer = ProtobufUtil.parseServerNameFrom(bytes);
+ return moveToServer.equals(actualServer);
+ });
+ TEST_UTIL.getAdmin().disableTable(tableName);
+ assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
deleted file mode 100644
index a5a98a6..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.client;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.ClusterMetrics.Option;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.StartMiniClusterOption;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
-import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
-import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
-import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
-import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-
-/**
- * Tests the scenarios where replicas are enabled for the meta table
- */
-@Category(LargeTests.class)
-public class TestMetaWithReplicas {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestMetaWithReplicas.class);
-
- private static final Logger LOG = LoggerFactory.getLogger(TestMetaWithReplicas.class);
- private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private static final int REGIONSERVERS_COUNT = 3;
-
- @Rule
- public TestName name = new TestName();
-
- @Before
- public void setup() throws Exception {
- TEST_UTIL.getConfiguration().setInt("zookeeper.session.timeout", 30000);
- TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
- TEST_UTIL.getConfiguration().setInt(
- StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
- StartMiniClusterOption option = StartMiniClusterOption.builder().
- numAlwaysStandByMasters(1).numMasters(1).numRegionServers(REGIONSERVERS_COUNT).build();
- TEST_UTIL.startMiniCluster(option);
- AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
- Set<ServerName> sns = new HashSet<ServerName>();
- ServerName hbaseMetaServerName =
- MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher());
- LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName);
- sns.add(hbaseMetaServerName);
- for (int replicaId = 1; replicaId < 3; replicaId++) {
- RegionInfo h = RegionReplicaUtil
- .getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId);
- AssignmentTestingUtil.waitForAssignment(am, h);
- ServerName sn = am.getRegionStates().getRegionServerOfRegion(h);
- assertNotNull(sn);
- LOG.info("HBASE:META DEPLOY: " + h.getRegionNameAsString() + " on " + sn);
- sns.add(sn);
- }
- // Fun. All meta region replicas have ended up on the one server. This will cause this test
- // to fail ... sometimes.
- if (sns.size() == 1) {
- int count = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
- assertTrue("count=" + count, count == REGIONSERVERS_COUNT);
- LOG.warn("All hbase:meta replicas are on the one server; moving hbase:meta: " + sns);
- int metaServerIndex = TEST_UTIL.getHBaseCluster().getServerWithMeta();
- int newServerIndex = metaServerIndex;
- while (newServerIndex == metaServerIndex) {
- newServerIndex = (newServerIndex + 1) % REGIONSERVERS_COUNT;
- }
- assertNotEquals(metaServerIndex, newServerIndex);
- ServerName destinationServerName =
- TEST_UTIL.getHBaseCluster().getRegionServer(newServerIndex).getServerName();
- ServerName metaServerName =
- TEST_UTIL.getHBaseCluster().getRegionServer(metaServerIndex).getServerName();
- assertNotEquals(destinationServerName, metaServerName);
- TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
- destinationServerName);
- }
- // Disable the balancer
- LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(),
- new Abortable() {
- AtomicBoolean aborted = new AtomicBoolean(false);
- @Override
- public boolean isAborted() {
- return aborted.get();
- }
- @Override
- public void abort(String why, Throwable e) {
- aborted.set(true);
- }
- });
- l.setBalancerOn(false);
- LOG.debug("All meta replicas assigned");
- }
-
- @After
- public void tearDown() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- @Test
- public void testMetaHTDReplicaCount() throws Exception {
- assertEquals(3,
- TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication());
- }
-
- @Test
- public void testZookeeperNodesForReplicas() throws Exception {
- // Checks all the znodes exist when meta's replicas are enabled
- ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
- Configuration conf = TEST_UTIL.getConfiguration();
- String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
- HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
- String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode,
- conf.get("zookeeper.znode.metaserver", "meta-region-server"));
- // check that the data in the znode is parseable (this would also mean the znode exists)
- byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
- ProtobufUtil.parseServerNameFrom(data);
- for (int i = 1; i < 3; i++) {
- String secZnode = ZNodePaths.joinZNode(baseZNode,
- conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
- String str = zkw.getZNodePaths().getZNodeForReplica(i);
- assertTrue(str.equals(secZnode));
- // check that the data in the znode is parseable (this would also mean the znode exists)
- data = ZKUtil.getData(zkw, secZnode);
- ProtobufUtil.parseServerNameFrom(data);
- }
- }
-
- @Test
- public void testShutdownHandling() throws Exception {
- // This test creates a table, flushes the meta (with 3 replicas), kills the
- // server holding the primary meta replica. Then it does a put/get into/from
- // the test table. The put/get operations would use the replicas to locate the
- // location of the test table's region
- shutdownMetaAndDoValidations(TEST_UTIL);
- }
-
- public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception {
- // This test creates a table, flushes the meta (with 3 replicas), kills the
- // server holding the primary meta replica. Then it does a put/get into/from
- // the test table. The put/get operations would use the replicas to locate the
- // location of the test table's region
- ZKWatcher zkw = util.getZooKeeperWatcher();
- Configuration conf = util.getConfiguration();
- conf.setBoolean(HConstants.USE_META_REPLICAS, true);
-
- String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
- HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
- String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode,
- conf.get("zookeeper.znode.metaserver", "meta-region-server"));
- byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
- ServerName primary = ProtobufUtil.parseServerNameFrom(data);
- LOG.info("Primary=" + primary.toString());
-
- TableName TABLE = TableName.valueOf("testShutdownHandling");
- byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
- if (util.getAdmin().tableExists(TABLE)) {
- util.getAdmin().disableTable(TABLE);
- util.getAdmin().deleteTable(TABLE);
- }
- byte[] row = Bytes.toBytes("test");
- ServerName master = null;
- try (Connection c = ConnectionFactory.createConnection(conf)) {
- try (Table htable = util.createTable(TABLE, FAMILIES)) {
- util.getAdmin().flush(TableName.META_TABLE_NAME);
- Thread.sleep(
- conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6);
- List<RegionInfo> regions = MetaTableAccessor.getTableRegions(c, TABLE);
- HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
- // Ensure that the primary server for test table is not the same one as the primary
- // of the meta region since we will be killing the srv holding the meta's primary...
- // We want to be able to write to the test table even when the meta is not present ..
- // If the servers are the same, then move the test table's region out of the server
- // to another random server
- if (hrl.getServerName().equals(primary)) {
- util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes());
- // wait for the move to complete
- do {
- Thread.sleep(10);
- hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
- } while (primary.equals(hrl.getServerName()));
- util.getAdmin().flush(TableName.META_TABLE_NAME);
- Thread.sleep(conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD,
- 30000) * 3);
- }
- // Ensure all metas are not on same hbase:meta replica=0 server!
-
- master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName();
- // kill the master so that regionserver recovery is not triggered at all
- // for the meta server
- LOG.info("Stopping master=" + master.toString());
- util.getHBaseClusterInterface().stopMaster(master);
- util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
- LOG.info("Master " + master + " stopped!");
- if (!master.equals(primary)) {
- util.getHBaseClusterInterface().killRegionServer(primary);
- util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
- }
- c.clearRegionLocationCache();
- }
- LOG.info("Running GETs");
- try (Table htable = c.getTable(TABLE)) {
- Put put = new Put(row);
- put.addColumn("foo".getBytes(), row, row);
- BufferedMutator m = c.getBufferedMutator(TABLE);
- m.mutate(put);
- m.flush();
- // Try to do a get of the row that was just put
- Result r = htable.get(new Get(row));
- assertTrue(Arrays.equals(r.getRow(), row));
- // now start back the killed servers and disable use of replicas. That would mean
- // calls go to the primary
- LOG.info("Starting Master");
- util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
- util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
- util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
- LOG.info("Master active!");
- c.clearRegionLocationCache();
- }
- }
- conf.setBoolean(HConstants.USE_META_REPLICAS, false);
- LOG.info("Running GETs no replicas");
- try (Connection c = ConnectionFactory.createConnection(conf)) {
- try (Table htable = c.getTable(TABLE)) {
- Result r = htable.get(new Get(row));
- assertTrue(Arrays.equals(r.getRow(), row));
- }
- }
- }
-
- @Test
- public void testAccessingUnknownTables() throws Exception {
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setBoolean(HConstants.USE_META_REPLICAS, true);
- Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf(name.getMethodName()));
- Get get = new Get(Bytes.toBytes("foo"));
- try {
- table.get(get);
- } catch (TableNotFoundException t) {
- return;
- }
- fail("Expected TableNotFoundException");
- }
-
- @Test
- public void testMetaAddressChange() throws Exception {
- // checks that even when the meta's location changes, the various
- // caches update themselves. Uses the master operations to test
- // this
- Configuration conf = TEST_UTIL.getConfiguration();
- ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
- String baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT,
- HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
- String primaryMetaZnode = ZNodePaths.joinZNode(baseZNode,
- conf.get("zookeeper.znode.metaserver", "meta-region-server"));
- // check that the data in the znode is parseable (this would also mean the znode exists)
- byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
- ServerName currentServer = ProtobufUtil.parseServerNameFrom(data);
- Collection<ServerName> liveServers = TEST_UTIL.getAdmin()
- .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet();
- ServerName moveToServer = null;
- for (ServerName s : liveServers) {
- if (!currentServer.equals(s)) {
- moveToServer = s;
- }
- }
- assertNotNull(moveToServer);
- final TableName tableName = TableName.valueOf(name.getMethodName());
- TEST_UTIL.createTable(tableName, "f");
- assertTrue(TEST_UTIL.getAdmin().tableExists(tableName));
- TEST_UTIL.getAdmin().move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
- moveToServer);
- int i = 0;
- assertNotEquals(currentServer, moveToServer);
- LOG.info("CurrentServer=" + currentServer + ", moveToServer=" + moveToServer);
- final int max = 10000;
- do {
- Thread.sleep(10);
- data = ZKUtil.getData(zkw, primaryMetaZnode);
- currentServer = ProtobufUtil.parseServerNameFrom(data);
- i++;
- } while (!moveToServer.equals(currentServer) && i < max); //wait for 10 seconds overall
- assertNotEquals(max, i);
- TEST_UTIL.getAdmin().disableTable(tableName);
- assertTrue(TEST_UTIL.getAdmin().isTableDisabled(tableName));
- }
-
- @Test
- public void testShutdownOfReplicaHolder() throws Exception {
- // checks that the when the server holding meta replica is shut down, the meta replica
- // can be recovered
- try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
- RegionLocator locator = conn.getRegionLocator(TableName.META_TABLE_NAME)) {
- HRegionLocation hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
- ServerName oldServer = hrl.getServerName();
- TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer);
- int i = 0;
- do {
- LOG.debug("Waiting for the replica " + hrl.getRegion() + " to come up");
- Thread.sleep(10000); // wait for the detection/recovery
- hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
- i++;
- } while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3);
- assertNotEquals(3, i);
- }
- }
-
- @Test
- public void testFailedReplicaAssigment() throws InterruptedException, IOException {
- //using our rigged master, to force a failed meta replica assignment
- TEST_UTIL.getMiniHBaseCluster().getConfiguration().setClass(HConstants.MASTER_IMPL, BrokenMetaReplicaMaster.class, HMaster.class);
- TEST_UTIL.getMiniHBaseCluster().stopMaster(0).join();
- HMaster newMaster = TEST_UTIL.getMiniHBaseCluster().startMaster().getMaster();
- //waiting for master to come up
- TEST_UTIL.waitFor(30000, () -> newMaster.isInitialized());
- TEST_UTIL.getMiniHBaseCluster().getConfiguration().unset(HConstants.MASTER_IMPL);
-
-
- AssignmentManager am = newMaster.getAssignmentManager();
- //showing one of the replicas got assigned
- RegionInfo metaReplicaHri = RegionReplicaUtil.getRegionInfoForReplica(
- RegionInfoBuilder.FIRST_META_REGIONINFO, 1);
- RegionStateNode metaReplicaRegionNode = am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri);
- Assert.assertNotNull(metaReplicaRegionNode.getRegionLocation());
- //showing one of the replicas failed to be assigned
- RegionInfo metaReplicaHri2 = RegionReplicaUtil.getRegionInfoForReplica(
- RegionInfoBuilder.FIRST_META_REGIONINFO, 2);
- RegionStateNode metaReplicaRegionNode2 = am.getRegionStates().getOrCreateRegionStateNode(metaReplicaHri2);
- Assert.assertNull(metaReplicaRegionNode2.getRegionLocation());
-
- //showing master is active and running
- Assert.assertFalse(newMaster.isStopping());
- Assert.assertFalse(newMaster.isStopped());
- Assert.assertTrue(newMaster.isActiveMaster());
- }
-
- public static class BrokenTransitRegionStateProcedure extends TransitRegionStateProcedure {
- protected BrokenTransitRegionStateProcedure() {
- //super(env, hri, assignCandidate, forceNewPlan, type);
- super(null, null, null, false,TransitionType.ASSIGN);
- }
- }
-
- public static class BrokenMetaReplicaMaster extends HMaster{
- public BrokenMetaReplicaMaster(final Configuration conf) throws IOException {
- super(conf);
- }
-
- @Override
- public AssignmentManager createAssignmentManager(MasterServices master) {
- return new BrokenMasterMetaAssignmentManager(master);
- }
- }
-
- public static class BrokenMasterMetaAssignmentManager extends AssignmentManager{
- MasterServices master;
- public BrokenMasterMetaAssignmentManager(final MasterServices master) {
- super(master);
- this.master = master;
- }
-
- public Future<byte[]> assignAsync(RegionInfo regionInfo, ServerName sn) throws IOException {
- RegionStateNode regionNode = getRegionStates().getOrCreateRegionStateNode(regionInfo);
- if (regionNode.getRegionInfo().getReplicaId() == 2) {
- regionNode.setProcedure(new BrokenTransitRegionStateProcedure());
- }
- return super.assignAsync(regionInfo, sn);
- }
- }
-}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java
new file mode 100644
index 0000000..1952925
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasBasic.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestMetaWithReplicasBasic extends MetaWithReplicasTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMetaWithReplicasBasic.class);
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ startCluster();
+ }
+
+ @Test
+ public void testMetaHTDReplicaCount() throws Exception {
+ assertEquals(3,
+ TEST_UTIL.getAdmin().getDescriptor(TableName.META_TABLE_NAME).getRegionReplication());
+ }
+
+ @Test
+ public void testZookeeperNodesForReplicas() throws Exception {
+ // Checks all the znodes exist when meta's replicas are enabled
+ ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
+ Configuration conf = TEST_UTIL.getConfiguration();
+ String baseZNode =
+ conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+ String primaryMetaZnode =
+ ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
+ // check that the data in the znode is parseable (this would also mean the znode exists)
+ byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
+ ProtobufUtil.parseServerNameFrom(data);
+ for (int i = 1; i < 3; i++) {
+ String secZnode = ZNodePaths.joinZNode(baseZNode,
+ conf.get("zookeeper.znode.metaserver", "meta-region-server") + "-" + i);
+ String str = zkw.getZNodePaths().getZNodeForReplica(i);
+ assertTrue(str.equals(secZnode));
+ // check that the data in the znode is parseable (this would also mean the znode exists)
+ data = ZKUtil.getData(zkw, secZnode);
+ ProtobufUtil.parseServerNameFrom(data);
+ }
+ }
+
+ @Test
+ public void testAccessingUnknownTables() throws Exception {
+ Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+ conf.setBoolean(HConstants.USE_META_REPLICAS, true);
+ Table table = TEST_UTIL.getConnection().getTable(name.getTableName());
+ Get get = new Get(Bytes.toBytes("foo"));
+ assertThrows(TableNotFoundException.class, () -> table.get(get));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java
new file mode 100644
index 0000000..036d00c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicasShutdownHandling.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestMetaWithReplicasShutdownHandling extends MetaWithReplicasTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMetaWithReplicasShutdownHandling.class);
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestMetaWithReplicasShutdownHandling.class);
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ startCluster();
+ }
+
+ @Test
+ public void testShutdownHandling() throws Exception {
+ // This test creates a table, flushes the meta (with 3 replicas), kills the
+ // server holding the primary meta replica. Then it does a put/get into/from
+ // the test table. The put/get operations would use the replicas to locate the
+ // location of the test table's region
+ shutdownMetaAndDoValidations(TEST_UTIL);
+ }
+
+ public static void shutdownMetaAndDoValidations(HBaseTestingUtility util) throws Exception {
+ // This test creates a table, flushes the meta (with 3 replicas), kills the
+ // server holding the primary meta replica. Then it does a put/get into/from
+ // the test table. The put/get operations would use the replicas to locate the
+ // location of the test table's region
+ ZKWatcher zkw = util.getZooKeeperWatcher();
+ Configuration conf = util.getConfiguration();
+ conf.setBoolean(HConstants.USE_META_REPLICAS, true);
+
+ String baseZNode =
+ conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT);
+ String primaryMetaZnode =
+ ZNodePaths.joinZNode(baseZNode, conf.get("zookeeper.znode.metaserver", "meta-region-server"));
+ byte[] data = ZKUtil.getData(zkw, primaryMetaZnode);
+ ServerName primary = ProtobufUtil.parseServerNameFrom(data);
+ LOG.info("Primary=" + primary.toString());
+
+ TableName TABLE = TableName.valueOf("testShutdownHandling");
+ byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
+ if (util.getAdmin().tableExists(TABLE)) {
+ util.getAdmin().disableTable(TABLE);
+ util.getAdmin().deleteTable(TABLE);
+ }
+ byte[] row = Bytes.toBytes("test");
+ ServerName master = null;
+ try (Connection c = ConnectionFactory.createConnection(util.getConfiguration())) {
+ try (Table htable = util.createTable(TABLE, FAMILIES)) {
+ util.getAdmin().flush(TableName.META_TABLE_NAME);
+ Thread.sleep(
+ conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 6);
+ List<RegionInfo> regions = MetaTableAccessor.getTableRegions(c, TABLE);
+ HRegionLocation hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
+ // Ensure that the primary server for test table is not the same one as the primary
+ // of the meta region since we will be killing the srv holding the meta's primary...
+ // We want to be able to write to the test table even when the meta is not present ..
+ // If the servers are the same, then move the test table's region out of the server
+ // to another random server
+ if (hrl.getServerName().equals(primary)) {
+ util.getAdmin().move(hrl.getRegion().getEncodedNameAsBytes());
+ // wait for the move to complete
+ do {
+ Thread.sleep(10);
+ hrl = MetaTableAccessor.getRegionLocation(c, regions.get(0));
+ } while (primary.equals(hrl.getServerName()));
+ util.getAdmin().flush(TableName.META_TABLE_NAME);
+ Thread.sleep(
+ conf.getInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 30000) * 3);
+ }
+ // Ensure all metas are not on same hbase:meta replica=0 server!
+
+ master = util.getHBaseClusterInterface().getClusterMetrics().getMasterName();
+ // kill the master so that regionserver recovery is not triggered at all
+ // for the meta server
+ LOG.info("Stopping master=" + master.toString());
+ util.getHBaseClusterInterface().stopMaster(master);
+ util.getHBaseClusterInterface().waitForMasterToStop(master, 60000);
+ LOG.info("Master " + master + " stopped!");
+ if (!master.equals(primary)) {
+ util.getHBaseClusterInterface().killRegionServer(primary);
+ util.getHBaseClusterInterface().waitForRegionServerToStop(primary, 60000);
+ }
+ c.clearRegionLocationCache();
+ }
+ LOG.info("Running GETs");
+ try (Table htable = c.getTable(TABLE)) {
+ Put put = new Put(row);
+ put.addColumn(Bytes.toBytes("foo"), row, row);
+ BufferedMutator m = c.getBufferedMutator(TABLE);
+ m.mutate(put);
+ m.flush();
+ // Try to do a get of the row that was just put
+ Result r = htable.get(new Get(row));
+ assertTrue(Arrays.equals(r.getRow(), row));
+ // now start back the killed servers and disable use of replicas. That would mean
+ // calls go to the primary
+ LOG.info("Starting Master");
+ util.getHBaseClusterInterface().startMaster(master.getHostname(), 0);
+ util.getHBaseClusterInterface().startRegionServer(primary.getHostname(), 0);
+ util.getHBaseClusterInterface().waitForActiveAndReadyMaster();
+ LOG.info("Master active!");
+ c.clearRegionLocationCache();
+ }
+ }
+ conf.setBoolean(HConstants.USE_META_REPLICAS, false);
+ LOG.info("Running GETs no replicas");
+ try (Connection c = ConnectionFactory.createConnection(conf);
+ Table htable = c.getTable(TABLE)) {
+ Result r = htable.get(new Get(row));
+ assertArrayEquals(row, r.getRow());
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java
new file mode 100644
index 0000000..11e5404
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestShutdownOfMetaReplicaHolder.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestShutdownOfMetaReplicaHolder extends MetaWithReplicasTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestShutdownOfMetaReplicaHolder.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestShutdownOfMetaReplicaHolder.class);
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ startCluster();
+ }
+
+ @Test
+ public void testShutdownOfReplicaHolder() throws Exception {
+ // checks that the when the server holding meta replica is shut down, the meta replica
+ // can be recovered
+ try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
+ RegionLocator locator = conn.getRegionLocator(TableName.META_TABLE_NAME)) {
+ HRegionLocation hrl = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
+ ServerName oldServer = hrl.getServerName();
+ TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer);
+ LOG.debug("Waiting for the replica {} to come up", hrl.getRegion());
+ TEST_UTIL.waitFor(30000, () -> {
+ HRegionLocation loc = locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1);
+ return loc != null && !loc.getServerName().equals(oldServer);
+ });
+ LOG.debug("Replica {} is online on {}, old server is {}", hrl.getRegion(),
+ locator.getRegionLocations(HConstants.EMPTY_START_ROW, true).get(1).getServerName(),
+ oldServer);
+ }
+ }
+}