You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2014/08/07 01:22:46 UTC
[02/10] HBASE-11611 Clean up ZK-based region assignment
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java
deleted file mode 100644
index fe992b6..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.migration;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.security.access.AccessControlLists;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test upgrade from no namespace in 0.94 to namespace directory structure.
- * Mainly tests that tables are migrated and consistent. Also verifies
- * that snapshots have been migrated correctly.
- *
- * <p>Uses a tarball which is an image of an 0.94 hbase.rootdir.
- *
- * <p>Contains tables with currentKeys as the stored keys:
- * foo, ns1.foo, ns2.foo
- *
- * <p>Contains snapshots with snapshot{num}Keys as the contents:
- * snapshot1Keys, snapshot2Keys
- *
- * Image also contains _acl_ table with one region and two storefiles.
- * This is needed to test the acl table migration.
- *
- */
-@Category(MediumTests.class)
-public class TestNamespaceUpgrade {
- static final Log LOG = LogFactory.getLog(TestNamespaceUpgrade.class);
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final static String snapshot1Keys[] =
- {"1","10","2","3","4","5","6","7","8","9"};
- private final static String snapshot2Keys[] =
- {"1","2","3","4","5","6","7","8","9"};
- private final static String currentKeys[] =
- {"1","2","3","4","5","6","7","8","9","A"};
- private final static TableName tables[] =
- {TableName.valueOf("foo"), TableName.valueOf("ns1.foo"), TableName.valueOf("ns.two.foo")};
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- // Start up our mini cluster on top of an 0.94 root.dir that has data from
- // a 0.94 hbase run and see if we can migrate to 0.96
- TEST_UTIL.startMiniZKCluster();
- TEST_UTIL.startMiniDFSCluster(1);
- Path testdir = TEST_UTIL.getDataTestDir("TestNamespaceUpgrade");
- // Untar our test dir.
- File untar = untar(new File(testdir.toString()));
- // Now copy the untar up into hdfs so when we start hbase, we'll run from it.
- Configuration conf = TEST_UTIL.getConfiguration();
- conf.setBoolean("hbase.assignment.usezk", true);
- FsShell shell = new FsShell(conf);
- FileSystem fs = FileSystem.get(conf);
- // find where hbase will root itself, so we can copy filesystem there
- Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
- if (!fs.isDirectory(hbaseRootDir.getParent())) {
- // mkdir at first
- fs.mkdirs(hbaseRootDir.getParent());
- }
- if(org.apache.hadoop.util.VersionInfo.getVersion().startsWith("2.")) {
- LOG.info("Hadoop version is 2.x, pre-migrating snapshot dir");
- FileSystem localFS = FileSystem.getLocal(conf);
- if(!localFS.rename(new Path(untar.toString(), HConstants.OLD_SNAPSHOT_DIR_NAME),
- new Path(untar.toString(), HConstants.SNAPSHOT_DIR_NAME))) {
- throw new IllegalStateException("Failed to move snapshot dir to 2.x expectation");
- }
- }
- doFsCommand(shell,
- new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
- doFsCommand(shell, new String [] {"-lsr", "/"});
- // See whats in minihdfs.
- Configuration toolConf = TEST_UTIL.getConfiguration();
- conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDefaultRootDirPath().toString());
- ToolRunner.run(toolConf, new NamespaceUpgrade(), new String[]{"--upgrade"});
- assertTrue(FSUtils.getVersion(fs, hbaseRootDir).equals(HConstants.FILE_SYSTEM_VERSION));
- doFsCommand(shell, new String [] {"-lsr", "/"});
- TEST_UTIL.startMiniHBaseCluster(1, 1);
-
- for(TableName table: tables) {
- int count = 0;
- for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new Scan())) {
- assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
- }
- Assert.assertEquals(currentKeys.length, count);
- }
- assertEquals(2, TEST_UTIL.getHBaseAdmin().listNamespaceDescriptors().length);
-
- //verify ACL table is migrated
- HTable secureTable = new HTable(conf, AccessControlLists.ACL_TABLE_NAME);
- ResultScanner scanner = secureTable.getScanner(new Scan());
- int count = 0;
- for(Result r : scanner) {
- count++;
- }
- assertEquals(3, count);
- assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(TableName.valueOf("_acl_")));
-
- //verify ACL table was compacted
- List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(secureTable.getName());
- for(HRegion region : regions) {
- assertEquals(1, region.getStores().size());
- }
- }
-
- static File untar(final File testdir) throws IOException {
- // Find the src data under src/test/data
- final String datafile = "TestNamespaceUpgrade";
- File srcTarFile = new File(
- System.getProperty("project.build.testSourceDirectory", "src/test") +
- File.separator + "data" + File.separator + datafile + ".tgz");
- File homedir = new File(testdir.toString());
- File tgtUntarDir = new File(homedir, "hbase");
- if (tgtUntarDir.exists()) {
- if (!FileUtil.fullyDelete(tgtUntarDir)) {
- throw new IOException("Failed delete of " + tgtUntarDir.toString());
- }
- }
- if (!srcTarFile.exists()) {
- throw new IOException(srcTarFile+" does not exist");
- }
- LOG.info("Untarring " + srcTarFile + " into " + homedir.toString());
- FileUtil.unTar(srcTarFile, homedir);
- Assert.assertTrue(tgtUntarDir.exists());
- return tgtUntarDir;
- }
-
- private static void doFsCommand(final FsShell shell, final String [] args)
- throws Exception {
- // Run the 'put' command.
- int errcode = shell.run(args);
- if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- @Test (timeout=300000)
- public void testSnapshots() throws IOException, InterruptedException {
- String snapshots[][] = {snapshot1Keys, snapshot2Keys};
- for(int i = 1; i <= snapshots.length; i++) {
- for(TableName table: tables) {
- TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot"+i, TableName.valueOf(table+"_clone"+i));
- FSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()),
- FSUtils.getRootDir(TEST_UTIL.getConfiguration()),
- LOG);
- int count = 0;
- for(Result res: new HTable(TEST_UTIL.getConfiguration(), table+"_clone"+i).getScanner(new
- Scan())) {
- assertEquals(snapshots[i-1][count++], Bytes.toString(res.getRow()));
- }
- Assert.assertEquals(table+"_snapshot"+i, snapshots[i-1].length, count);
- }
- }
- }
-
- @Test (timeout=300000)
- public void testRenameUsingSnapshots() throws Exception {
- String newNS = "newNS";
- TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(newNS).build());
- for(TableName table: tables) {
- int count = 0;
- for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new
- Scan())) {
- assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
- }
- TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot3", table);
- final TableName newTableName =
- TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3");
- TEST_UTIL.getHBaseAdmin().cloneSnapshot(table + "_snapshot3", newTableName);
- Thread.sleep(1000);
- count = 0;
- for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
- Scan())) {
- assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
- }
- FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath()
- , LOG);
- Assert.assertEquals(newTableName + "", currentKeys.length, count);
- TEST_UTIL.getHBaseAdmin().flush(newTableName.toBytes());
- TEST_UTIL.getHBaseAdmin().majorCompact(newTableName.toBytes());
- TEST_UTIL.waitFor(30000, new Waiter.Predicate<IOException>() {
- @Override
- public boolean evaluate() throws IOException {
- try {
- return TEST_UTIL.getHBaseAdmin().getCompactionState(newTableName.toBytes()) ==
- AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
- });
- }
-
- String nextNS = "nextNS";
- TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(nextNS).build());
- for(TableName table: tables) {
- TableName srcTable = TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3");
- TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot4", srcTable);
- TableName newTableName =
- TableName.valueOf(nextNS + TableName.NAMESPACE_DELIM + table + "_clone4");
- TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot4", newTableName);
- FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(),
- LOG);
- int count = 0;
- for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new
- Scan())) {
- assertEquals(currentKeys[count++], Bytes.toString(res.getRow()));
- }
- Assert.assertEquals(newTableName + "", currentKeys.length, count);
- }
- }
-
- @Test (timeout=300000)
- public void testOldDirsAreGonePostMigration() throws IOException {
- FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
- Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
- List <String> dirs = new ArrayList<String>(NamespaceUpgrade.NON_USER_TABLE_DIRS);
- // Remove those that are not renamed
- dirs.remove(HConstants.HBCK_SIDELINEDIR_NAME);
- dirs.remove(HConstants.SNAPSHOT_DIR_NAME);
- dirs.remove(HConstants.HBASE_TEMP_DIRECTORY);
- for (String dir: dirs) {
- assertFalse(fs.exists(new Path(hbaseRootDir, dir)));
- }
- }
-
- @Test (timeout=300000)
- public void testNewDirsArePresentPostMigration() throws IOException {
- FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
- // Below list does not include 'corrupt' because there is no 'corrupt' in the tgz
- String [] newdirs = new String [] {HConstants.BASE_NAMESPACE_DIR,
- HConstants.HREGION_LOGDIR_NAME};
- Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
- for (String dir: newdirs) {
- assertTrue(dir, fs.exists(new Path(hbaseRootDir, dir)));
- }
- }
-
- @Test (timeout = 300000)
- public void testACLTableMigration() throws IOException {
- Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testACLTable");
- FileSystem fs = TEST_UTIL.getTestFileSystem();
- Configuration conf = TEST_UTIL.getConfiguration();
- byte[] FAMILY = Bytes.toBytes("l");
- byte[] QUALIFIER = Bytes.toBytes("testUser");
- byte[] VALUE = Bytes.toBytes("RWCA");
-
- // Create a Region
- HTableDescriptor aclTable = new HTableDescriptor(TableName.valueOf("testACLTable"));
- aclTable.addFamily(new HColumnDescriptor(FAMILY));
- FSTableDescriptors fstd = new FSTableDescriptors(fs, rootDir);
- fstd.createTableDescriptor(aclTable);
- HRegionInfo hriAcl = new HRegionInfo(aclTable.getTableName(), null, null);
- HRegion region = HRegion.createHRegion(hriAcl, rootDir, conf, aclTable);
- try {
- // Create rows
- Put p = new Put(Bytes.toBytes("-ROOT-"));
- p.addImmutable(FAMILY, QUALIFIER, VALUE);
- region.put(p);
- p = new Put(Bytes.toBytes(".META."));
- p.addImmutable(FAMILY, QUALIFIER, VALUE);
- region.put(p);
- p = new Put(Bytes.toBytes("_acl_"));
- p.addImmutable(FAMILY, QUALIFIER, VALUE);
- region.put(p);
-
- NamespaceUpgrade upgrade = new NamespaceUpgrade();
- upgrade.updateAcls(region);
-
- // verify rows -ROOT- is removed
- Get g = new Get(Bytes.toBytes("-ROOT-"));
- Result r = region.get(g);
- assertTrue(r == null || r.size() == 0);
-
- // verify rows _acl_ is renamed to hbase:acl
- g = new Get(AccessControlLists.ACL_TABLE_NAME.toBytes());
- r = region.get(g);
- assertTrue(r != null && r.size() == 1);
- assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0);
-
- // verify rows .META. is renamed to hbase:meta
- g = new Get(TableName.META_TABLE_NAME.toBytes());
- r = region.get(g);
- assertTrue(r != null && r.size() == 1);
- assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0);
- } finally {
- region.close();
- // Delete the region
- HRegionFileSystem.deleteRegionFromFileSystem(conf, fs,
- FSUtils.getTableDir(rootDir, hriAcl.getTable()), hriAcl);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java
deleted file mode 100644
index ab293f2..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.migration;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.io.FileLink;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
-import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
-import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.HFileV1Detector;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.zookeeper.KeeperException;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * Upgrade to 0.96 involves detecting HFileV1 in existing cluster, updating namespace and
- * updating znodes. This class tests for HFileV1 detection and upgrading znodes.
- * Uprading namespace is tested in {@link TestNamespaceUpgrade}.
- */
-@Category(MediumTests.class)
-public class TestUpgradeTo96 {
-
- static final Log LOG = LogFactory.getLog(TestUpgradeTo96.class);
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
- /**
- * underlying file system instance
- */
- private static FileSystem fs;
- /**
- * hbase root dir
- */
- private static Path hbaseRootDir;
- private static ZooKeeperWatcher zkw;
- /**
- * replication peer znode (/hbase/replication/peers)
- */
- private static String replicationPeerZnode;
- /**
- * znode of a table
- */
- private static String tableAZnode;
- private static ReplicationPeer peer1;
- /**
- * znode for replication peer1 (/hbase/replication/peers/1)
- */
- private static String peer1Znode;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- // Start up the mini cluster on top of an 0.94 root.dir that has data from
- // a 0.94 hbase run and see if we can migrate to 0.96
- TEST_UTIL.startMiniZKCluster();
- TEST_UTIL.startMiniDFSCluster(1);
-
- hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
- fs = FileSystem.get(TEST_UTIL.getConfiguration());
- FSUtils.setRootDir(TEST_UTIL.getConfiguration(), hbaseRootDir);
- zkw = TEST_UTIL.getZooKeeperWatcher();
-
- Path testdir = TEST_UTIL.getDataTestDir("TestUpgradeTo96");
- // get the untar 0.94 file structure
-
- set94FSLayout(testdir);
- setUp94Znodes();
- }
-
- /**
- * Lays out 0.94 file system layout using {@link TestNamespaceUpgrade} apis.
- * @param testdir
- * @throws IOException
- * @throws Exception
- */
- private static void set94FSLayout(Path testdir) throws IOException, Exception {
- File untar = TestNamespaceUpgrade.untar(new File(testdir.toString()));
- if (!fs.exists(hbaseRootDir.getParent())) {
- // mkdir at first
- fs.mkdirs(hbaseRootDir.getParent());
- }
- FsShell shell = new FsShell(TEST_UTIL.getConfiguration());
- shell.run(new String[] { "-put", untar.toURI().toString(), hbaseRootDir.toString() });
- // See whats in minihdfs.
- shell.run(new String[] { "-lsr", "/" });
- }
-
- /**
- * Sets znodes used in 0.94 version. Only table and replication znodes will be upgraded to PB,
- * others would be deleted.
- * @throws KeeperException
- */
- private static void setUp94Znodes() throws IOException, KeeperException {
- // add some old znodes, which would be deleted after upgrade.
- String rootRegionServerZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server");
- ZKUtil.createWithParents(zkw, rootRegionServerZnode);
- ZKUtil.createWithParents(zkw, zkw.backupMasterAddressesZNode);
- // add table znode, data of its children would be protobuffized
- tableAZnode = ZKUtil.joinZNode(zkw.tableZNode, "a");
- ZKUtil.createWithParents(zkw, tableAZnode,
- Bytes.toBytes(ZooKeeperProtos.Table.State.ENABLED.toString()));
- // add replication znodes, data of its children would be protobuffized
- String replicationZnode = ZKUtil.joinZNode(zkw.baseZNode, "replication");
- replicationPeerZnode = ZKUtil.joinZNode(replicationZnode, "peers");
- peer1Znode = ZKUtil.joinZNode(replicationPeerZnode, "1");
- peer1 = ReplicationPeer.newBuilder().setClusterkey("abc:123:/hbase").build();
- ZKUtil.createWithParents(zkw, peer1Znode, Bytes.toBytes(peer1.getClusterkey()));
- }
-
- /**
- * Tests a 0.94 filesystem for any HFileV1.
- * @throws Exception
- */
- @Test
- public void testHFileV1Detector() throws Exception {
- assertEquals(0, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null));
- }
-
- /**
- * Creates a corrupt file, and run HFileV1 detector tool
- * @throws Exception
- */
- @Test
- public void testHFileV1DetectorWithCorruptFiles() throws Exception {
- // add a corrupt file.
- Path tablePath = new Path(hbaseRootDir, "foo");
- FileStatus[] regionsDir = fs.listStatus(tablePath);
- if (regionsDir == null) throw new IOException("No Regions found for table " + "foo");
- Path columnFamilyDir = null;
- Path targetRegion = null;
- for (FileStatus s : regionsDir) {
- if (fs.exists(new Path(s.getPath(), HRegionFileSystem.REGION_INFO_FILE))) {
- targetRegion = s.getPath();
- break;
- }
- }
- FileStatus[] cfs = fs.listStatus(targetRegion);
- for (FileStatus f : cfs) {
- if (f.isDirectory()) {
- columnFamilyDir = f.getPath();
- break;
- }
- }
- LOG.debug("target columnFamilyDir: " + columnFamilyDir);
- // now insert a corrupt file in the columnfamily.
- Path corruptFile = new Path(columnFamilyDir, "corrupt_file");
- if (!fs.createNewFile(corruptFile)) throw new IOException("Couldn't create corrupt file: "
- + corruptFile);
- assertEquals(1, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null));
- // remove the corrupt file
- FileSystem.get(TEST_UTIL.getConfiguration()).delete(corruptFile, false);
- }
-
- @Test
- public void testHFileLink() throws Exception {
- // pass a link, and verify that correct paths are returned.
- Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
- Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48");
- Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48");
- Path preNamespaceArchivePath =
- new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48");
- Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48");
- boolean preNSTablePathExists = false;
- boolean preNSArchivePathExists = false;
- boolean preNSTempPathExists = false;
- assertTrue(HFileLink.isHFileLink(aFileLink));
- HFileLink hFileLink = new HFileLink(TEST_UTIL.getConfiguration(), aFileLink);
- assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString()));
-
- HFileV1Detector t = new HFileV1Detector();
- t.setConf(TEST_UTIL.getConfiguration());
- FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink);
- //assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp) to look.
- assertTrue(fileLink.getLocations().length == 6);
- for (Path p : fileLink.getLocations()) {
- if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true;
- if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true;
- if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true;
- }
- assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists);
- }
-
- @Test
- public void testADirForHFileV1() throws Exception {
- Path tablePath = new Path(hbaseRootDir, "foo");
- System.out.println("testADirForHFileV1: " + tablePath.makeQualified(fs));
- System.out.println("Passed: " + hbaseRootDir + "/foo");
- assertEquals(0,
- ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), new String[] { "-p"
- + "foo" }));
- }
-
- @Test
- public void testZnodeMigration() throws Exception {
- String rootRSZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server");
- assertTrue(ZKUtil.checkExists(zkw, rootRSZnode) > -1);
- ToolRunner.run(TEST_UTIL.getConfiguration(), new UpgradeTo96(), new String[] { "-execute" });
- assertEquals(-1, ZKUtil.checkExists(zkw, rootRSZnode));
- byte[] data = ZKUtil.getData(zkw, tableAZnode);
- assertTrue(ProtobufUtil.isPBMagicPrefix(data));
- checkTableState(data, ZooKeeperProtos.Table.State.ENABLED);
- // ensure replication znodes are there, and protobuffed.
- data = ZKUtil.getData(zkw, peer1Znode);
- assertTrue(ProtobufUtil.isPBMagicPrefix(data));
- checkReplicationPeerData(data, peer1);
- }
-
- private void checkTableState(byte[] data, State expectedState)
- throws InvalidProtocolBufferException {
- ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder();
- int magicLen = ProtobufUtil.lengthOfPBMagic();
- ZooKeeperProtos.Table t = builder.mergeFrom(data, magicLen, data.length - magicLen).build();
- assertTrue(t.getState() == expectedState);
- }
-
- private void checkReplicationPeerData(byte[] data, ReplicationPeer peer)
- throws InvalidProtocolBufferException {
- int magicLen = ProtobufUtil.lengthOfPBMagic();
- ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder();
- assertEquals(builder.mergeFrom(data, magicLen, data.length - magicLen).build().getClusterkey(),
- peer.getClusterkey());
-
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniHBaseCluster();
- TEST_UTIL.shutdownMiniDFSCluster();
- TEST_UTIL.shutdownMiniZKCluster();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
index 8de605d..6cec1e2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java
@@ -34,16 +34,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.NotServingRegionException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -52,14 +51,13 @@ import org.apache.hadoop.hbase.client.MetaScanner;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
+import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.ConfigUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.StoppableImplementation;
@@ -74,6 +72,7 @@ import com.google.common.collect.Sets;
import com.google.protobuf.ServiceException;
@Category(LargeTests.class)
+@SuppressWarnings("deprecation")
public class TestEndToEndSplitTransaction {
private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class);
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -109,7 +108,6 @@ public class TestEndToEndSplitTransaction {
.getRegionName();
HRegion region = server.getRegion(regionName);
SplitTransaction split = new SplitTransaction(region, splitRow);
- split.useZKForAssignment = ConfigUtil.useZKForAssignment(conf);
split.prepare();
// 1. phase I
@@ -126,14 +124,10 @@ public class TestEndToEndSplitTransaction {
// 3. finish phase II
// note that this replicates some code from SplitTransaction
// 2nd daughter first
- if (split.useZKForAssignment) {
- server.postOpenDeployTasks(regions.getSecond());
- } else {
server.reportRegionStateTransition(
RegionServerStatusProtos.RegionStateTransition.TransitionCode.SPLIT,
region.getRegionInfo(), regions.getFirst().getRegionInfo(),
regions.getSecond().getRegionInfo());
- }
// Add to online regions
server.addToOnlineRegions(regions.getSecond());
@@ -143,21 +137,11 @@ public class TestEndToEndSplitTransaction {
// past splitkey is ok.
assertTrue(test(con, tableName, lastRow, server));
- // first daughter second
- if (split.useZKForAssignment) {
- server.postOpenDeployTasks(regions.getFirst());
- }
// Add to online regions
server.addToOnlineRegions(regions.getFirst());
assertTrue(test(con, tableName, firstRow, server));
assertTrue(test(con, tableName, lastRow, server));
- if (split.useZKForAssignment) {
- // 4. phase III
- ((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
- .getSplitTransactionCoordination().completeSplitTransaction(server, regions.getFirst(),
- regions.getSecond(), split.std, region);
- }
assertTrue(test(con, tableName, firstRow, server));
assertTrue(test(con, tableName, lastRow, server));
}
@@ -341,7 +325,7 @@ public class TestEndToEndSplitTransaction {
verifyStartEndKeys(keys);
//HTable.getRegionsInfo()
- Map<HRegionInfo, ServerName> regions = table.getRegionLocations();
+ Map<HRegionInfo, ServerName> regions = table.getRegionLocations();
verifyTableRegions(regions.keySet());
} finally {
IOUtils.closeQuietly(table);
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
index d9883e0..ac5e41e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
@@ -37,13 +37,12 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
-import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
@@ -94,7 +93,8 @@ public class TestRegionMergeTransactionOnCluster {
private static HMaster master;
private static Admin admin;
- static void setupOnce() throws Exception {
+ @BeforeClass
+ public static void beforeAllTests() throws Exception {
// Start a cluster
TEST_UTIL.startMiniCluster(NB_SERVERS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
@@ -103,13 +103,6 @@ public class TestRegionMergeTransactionOnCluster {
admin = TEST_UTIL.getHBaseAdmin();
}
- @BeforeClass
- public static void beforeAllTests() throws Exception {
- // Use ZK for region assignment
- TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
- setupOnce();
- }
-
@AfterClass
public static void afterAllTests() throws Exception {
TEST_UTIL.shutdownMiniCluster();
@@ -148,7 +141,7 @@ public class TestRegionMergeTransactionOnCluster {
}
// We should not be able to assign it again
- am.assign(hri, true, true);
+ am.assign(hri, true);
assertFalse("Merged region can't be assigned",
regionStates.isRegionInTransition(hri));
assertTrue(regionStates.isRegionInState(hri, State.MERGED));
@@ -162,6 +155,7 @@ public class TestRegionMergeTransactionOnCluster {
table.close();
}
+ @SuppressWarnings("deprecation")
@Test
public void testCleanMergeReference() throws Exception {
LOG.info("Starting testCleanMergeReference");
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
index 9f64a7c..a7067f1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
@@ -45,9 +45,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -94,26 +92,18 @@ public class TestRegionReplicas {
@AfterClass
public static void afterClass() throws Exception {
+ HRegionServer.TEST_SKIP_REPORTING_TRANSITION = false;
table.close();
HTU.shutdownMiniCluster();
}
- @After
- public void after() throws Exception {
- // Clean the state if the test failed before cleaning the znode
- // It does not manage all bad failures, so if there are multiple failures, only
- // the first one should be looked at.
- ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriPrimary);
- }
-
private HRegionServer getRS() {
return HTU.getMiniHBaseCluster().getRegionServer(0);
}
private void openRegion(HRegionInfo hri) throws Exception {
- ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
// first version is '0'
- AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null);
+ AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, null, null);
AdminProtos.OpenRegionResponse responseOpen = getRS().getRSRpcServices().openRegion(null, orr);
Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
Assert.assertTrue(responseOpen.getOpeningState(0).
@@ -122,16 +112,12 @@ public class TestRegionReplicas {
}
private void closeRegion(HRegionInfo hri) throws Exception {
- ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
-
AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(),
- hri.getEncodedName(), true);
+ hri.getEncodedName());
AdminProtos.CloseRegionResponse responseClose = getRS().getRSRpcServices().closeRegion(null, crr);
Assert.assertTrue(responseClose.getClosed());
checkRegionIsClosed(hri.getEncodedName());
-
- ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), getRS().getServerName());
}
private void checkRegionIsOpened(String encodedRegionName) throws Exception {
@@ -141,9 +127,6 @@ public class TestRegionReplicas {
}
Assert.assertTrue(getRS().getRegionByEncodedName(encodedRegionName).isAvailable());
-
- Assert.assertTrue(
- ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), encodedRegionName, getRS().getServerName()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
index 0843abe..3164f1d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
@@ -26,29 +26,22 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
-import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
-import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination;
-import org.apache.hadoop.hbase.executor.EventType;
+import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -77,7 +70,6 @@ public class TestRegionServerNoMaster {
@BeforeClass
public static void before() throws Exception {
- HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
HTU.startMiniCluster(NB_SERVERS);
final byte[] tableName = Bytes.toBytes(TestRegionServerNoMaster.class.getSimpleName());
@@ -94,27 +86,37 @@ public class TestRegionServerNoMaster {
}
public static void stopMasterAndAssignMeta(HBaseTestingUtility HTU)
- throws NodeExistsException, KeeperException, IOException, InterruptedException {
- // No master
- HTU.getHBaseCluster().getMaster().stopMaster();
+ throws IOException, InterruptedException {
+ // Stop master
+ HMaster master = HTU.getHBaseCluster().getMaster();
+ ServerName masterAddr = master.getServerName();
+ master.stopMaster();
Log.info("Waiting until master thread exits");
while (HTU.getHBaseCluster().getMasterThread() != null
&& HTU.getHBaseCluster().getMasterThread().isAlive()) {
Threads.sleep(100);
}
+
+ HRegionServer.TEST_SKIP_REPORTING_TRANSITION = true;
// Master is down, so is the meta. We need to assign it somewhere
// so that regions can be assigned during the mocking phase.
- HRegionServer hrs = HTU.getHBaseCluster().getRegionServer(0);
+ HRegionServer hrs = HTU.getHBaseCluster()
+ .getLiveRegionServerThreads().get(0).getRegionServer();
ZooKeeperWatcher zkw = hrs.getZooKeeper();
- ZKAssign.createNodeOffline(
- zkw, HRegionInfo.FIRST_META_REGIONINFO, hrs.getServerName());
+ MetaTableLocator mtl = new MetaTableLocator();
+ ServerName sn = mtl.getMetaRegionLocation(zkw);
+ if (sn != null && !masterAddr.equals(sn)) {
+ return;
+ }
+
ProtobufUtil.openRegion(hrs.getRSRpcServices(),
hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
- MetaTableLocator mtl = new MetaTableLocator();
while (true) {
- ServerName sn = mtl.getMetaRegionLocation(zkw);
- if (sn != null && sn.equals(hrs.getServerName())) {
+ sn = mtl.getMetaRegionLocation(zkw);
+ if (sn != null && sn.equals(hrs.getServerName())
+ && hrs.onlineRegions.containsKey(
+ HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
break;
}
Thread.sleep(100);
@@ -135,19 +137,11 @@ public class TestRegionServerNoMaster {
@AfterClass
public static void afterClass() throws Exception {
+ HRegionServer.TEST_SKIP_REPORTING_TRANSITION = false;
table.close();
HTU.shutdownMiniCluster();
}
- @After
- public void after() throws Exception {
- // Clean the state if the test failed before cleaning the znode
- // It does not manage all bad failures, so if there are multiple failures, only
- // the first one should be looked at.
- ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hri);
- }
-
-
private static HRegionServer getRS() {
return HTU.getHBaseCluster().getLiveRegionServerThreads().get(0).getRegionServer();
}
@@ -157,11 +151,8 @@ public class TestRegionServerNoMaster {
* Reopen the region. Reused in multiple tests as we always leave the region open after a test.
*/
private void reopenRegion() throws Exception {
- // We reopen. We need a ZK node here, as a open is always triggered by a master.
- ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
- // first version is '0'
AdminProtos.OpenRegionRequest orr =
- RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null);
+ RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, null, null);
AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr);
Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
Assert.assertTrue(responseOpen.getOpeningState(0).
@@ -178,10 +169,6 @@ public class TestRegionServerNoMaster {
}
Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
-
- Assert.assertTrue(
- ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
- getRS().getServerName()));
}
@@ -196,8 +183,6 @@ public class TestRegionServerNoMaster {
} catch (NotServingRegionException expected) {
// That's how it work: if the region is closed we have an exception.
}
-
- // We don't delete the znode here, because there is not always a znode.
}
@@ -207,7 +192,7 @@ public class TestRegionServerNoMaster {
private void closeNoZK() throws Exception {
// no transition in ZK
AdminProtos.CloseRegionRequest crr =
- RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false);
+ RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName);
AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
Assert.assertTrue(responseClose.getClosed());
@@ -222,42 +207,6 @@ public class TestRegionServerNoMaster {
reopenRegion();
}
- @Test(timeout = 60000)
- public void testCloseByMasterWithoutZNode() throws Exception {
-
- // Transition in ZK on. This should fail, as there is no znode
- AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(
- getRS().getServerName(), regionName, true);
- AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
- Assert.assertTrue(responseClose.getClosed());
-
- // now waiting. After a while, the transition should be done
- while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
- Thread.sleep(1);
- }
-
- // the region is still available, the close got rejected at the end
- Assert.assertTrue("The close should have failed", getRS().getRegion(regionName).isAvailable());
- }
-
- @Test(timeout = 60000)
- public void testOpenCloseByMasterWithZNode() throws Exception {
-
- ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
-
- AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(
- getRS().getServerName(), regionName, true);
- AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
- Assert.assertTrue(responseClose.getClosed());
-
- checkRegionIsClosed();
-
- ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
- getRS().getServerName());
-
- reopenRegion();
- }
-
/**
* Test that we can send multiple openRegion to the region server.
* This is used when:
@@ -275,13 +224,10 @@ public class TestRegionServerNoMaster {
closeNoZK();
checkRegionIsClosed();
- // We reopen. We need a ZK node here, as a open is always triggered by a master.
- ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
-
// We're sending multiple requests in a row. The region server must handle this nicely.
for (int i = 0; i < 10; i++) {
AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(
- getRS().getServerName(), hri, 0, null, null);
+ getRS().getServerName(), hri, null, null);
AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr);
Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
@@ -307,7 +253,7 @@ public class TestRegionServerNoMaster {
// fake region to be closing now, need to clear state afterwards
getRS().regionsInTransitionInRS.put(hri.getEncodedNameAsBytes(), Boolean.FALSE);
AdminProtos.OpenRegionRequest orr =
- RequestConverter.buildOpenRegionRequest(sn, hri, 0, null, null);
+ RequestConverter.buildOpenRegionRequest(sn, hri, null, null);
getRS().rpcServices.openRegion(null, orr);
Assert.fail("The closing region should not be opened");
} catch (ServiceException se) {
@@ -320,12 +266,9 @@ public class TestRegionServerNoMaster {
@Test(timeout = 60000)
public void testMultipleCloseFromMaster() throws Exception {
-
- // As opening, we must support multiple requests on the same region
- ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
for (int i = 0; i < 10; i++) {
AdminProtos.CloseRegionRequest crr =
- RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, 0, null, true);
+ RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, null);
try {
AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr);
Assert.assertEquals("The first request should succeeds", 0, i);
@@ -338,11 +281,6 @@ public class TestRegionServerNoMaster {
checkRegionIsClosed();
- Assert.assertTrue(
- ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
- getRS().getServerName())
- );
-
reopenRegion();
}
@@ -356,12 +294,11 @@ public class TestRegionServerNoMaster {
checkRegionIsClosed();
// Let do the initial steps, without having a handler
- ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);
// That's a close without ZK.
AdminProtos.CloseRegionRequest crr =
- RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false);
+ RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName);
try {
getRS().rpcServices.closeRegion(null, crr);
Assert.assertTrue(false);
@@ -375,89 +312,11 @@ public class TestRegionServerNoMaster {
// Let's start the open handler
HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());
- BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager();
- csm.initialize(getRS());
- csm.start();
-
- ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
- new ZkOpenRegionCoordination.ZkOpenRegionDetails();
- zkCrd.setServerName(getRS().getServerName());
- zkCrd.setVersionOfOfflineNode(0);
-
- getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd,
- csm.getOpenRegionCoordination(), zkCrd));
+ getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd));
// The open handler should have removed the region from RIT but kept the region closed
checkRegionIsClosed();
- // The open handler should have updated the value in ZK.
- Assert.assertTrue(ZKAssign.deleteNode(
- getRS().getZooKeeper(), hri.getEncodedName(),
- EventType.RS_ZK_REGION_FAILED_OPEN, 1)
- );
-
- reopenRegion();
- }
-
- /**
- * Test an open then a close with ZK. This is going to mess-up the ZK states, so
- * the opening will fail as well because it doesn't find what it expects in ZK.
- */
- @Test(timeout = 60000)
- public void testCancelOpeningWithZK() throws Exception {
- // We close
- closeNoZK();
- checkRegionIsClosed();
-
- // Let do the initial steps, without having a handler
- getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE);
-
- // That's a close without ZK.
- ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
- AdminProtos.CloseRegionRequest crr =
- RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false);
- try {
- getRS().rpcServices.closeRegion(null, crr);
- Assert.assertTrue(false);
- } catch (ServiceException expected) {
- Assert.assertTrue(expected.getCause() instanceof NotServingRegionException);
- }
-
- // The close should have left the ZK state as it is: it's the job the AM to delete it
- Assert.assertTrue(ZKAssign.deleteNode(
- getRS().getZooKeeper(), hri.getEncodedName(),
- EventType.M_ZK_REGION_CLOSING, 0)
- );
-
- // The state in RIT should have changed to close
- Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get(
- hri.getEncodedNameAsBytes()));
-
- // Let's start the open handler
- // It should not succeed for two reasons:
- // 1) There is no ZK node
- // 2) The region in RIT was changed.
- // The order is more or less implementation dependant.
- HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable());
-
- BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager();
- csm.initialize(getRS());
- csm.start();
-
- ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd =
- new ZkOpenRegionCoordination.ZkOpenRegionDetails();
- zkCrd.setServerName(getRS().getServerName());
- zkCrd.setVersionOfOfflineNode(0);
-
- getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd,
- csm.getOpenRegionCoordination(), zkCrd));
-
- // The open handler should have removed the region from RIT but kept the region closed
- checkRegionIsClosed();
-
- // We should not find any znode here.
- Assert.assertEquals(-1, ZKAssign.getVersion(HTU.getZooKeeperWatcher(), hri));
-
reopenRegion();
}
@@ -473,7 +332,7 @@ public class TestRegionServerNoMaster {
ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1);
try {
- CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(earlierServerName, regionName, true);
+ CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(earlierServerName, regionName);
getRS().getRSRpcServices().closeRegion(null, request);
Assert.fail("The closeRegion should have been rejected");
} catch (ServiceException se) {
@@ -485,7 +344,7 @@ public class TestRegionServerNoMaster {
closeNoZK();
try {
AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(
- earlierServerName, hri, 0, null, null);
+ earlierServerName, hri, null, null);
getRS().getRSRpcServices().openRegion(null, orr);
Assert.fail("The openRegion should have been rejected");
} catch (ServiceException se) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index da4b3d9..3818b60 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -48,15 +47,14 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.RegionTransition;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -68,14 +66,10 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro;
-import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination;
-import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination;
import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.RegionState;
@@ -90,12 +84,9 @@ import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hbase.zookeeper.ZKAssign;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.apache.zookeeper.data.Stat;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -120,26 +111,15 @@ public class TestSplitTransactionOnCluster {
private static final int NB_SERVERS = 3;
private static CountDownLatch latch = new CountDownLatch(1);
private static volatile boolean secondSplit = false;
- private static volatile boolean callRollBack = false;
- private static volatile boolean firstSplitCompleted = false;
- private static boolean useZKForAssignment = true;
static final HBaseTestingUtility TESTING_UTIL =
new HBaseTestingUtility();
- static void setupOnce() throws Exception {
+ @BeforeClass public static void before() throws Exception {
TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
- useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean(
- "hbase.assignment.usezk", false);
TESTING_UTIL.startMiniCluster(NB_SERVERS);
}
- @BeforeClass public static void before() throws Exception {
- // Use ZK for region assignment
- TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
- setupOnce();
- }
-
@AfterClass public static void after() throws Exception {
TESTING_UTIL.shutdownMiniCluster();
}
@@ -181,100 +161,6 @@ public class TestSplitTransactionOnCluster {
return hri;
}
- @SuppressWarnings("deprecation")
- @Test(timeout = 60000)
- public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception {
- final TableName tableName =
- TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
-
- if (!useZKForAssignment) {
- // This test doesn't apply if not using ZK for assignment
- return;
- }
-
- try {
- // Create table then get the single region for our new table.
- HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
- final List<HRegion> regions = cluster.getRegions(tableName);
- HRegionInfo hri = getAndCheckSingleTableRegion(regions);
- int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName());
- final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);
- insertData(tableName.getName(), admin, t);
- t.close();
-
- // Turn off balancer so it doesn't cut in and mess up our placements.
- this.admin.setBalancerRunning(false, true);
- // Turn off the meta scanner so it don't remove parent on us.
- cluster.getMaster().setCatalogJanitorEnabled(false);
-
- // find a splittable region
- final HRegion region = findSplittableRegion(regions);
- assertTrue("not able to find a splittable region", region != null);
- MockedCoordinatedStateManager cp = new MockedCoordinatedStateManager();
- cp.initialize(regionServer, region);
- cp.start();
- regionServer.csm = cp;
-
- new Thread() {
- @Override
- public void run() {
- SplitTransaction st = null;
- st = new MockedSplitTransaction(region, Bytes.toBytes("row2"));
- try {
- st.prepare();
- st.execute(regionServer, regionServer);
- } catch (IOException e) {
-
- }
- }
- }.start();
- for (int i = 0; !callRollBack && i < 100; i++) {
- Thread.sleep(100);
- }
- assertTrue("Waited too long for rollback", callRollBack);
- SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3"));
- try {
- secondSplit = true;
- // make region splittable
- region.initialize();
- st.prepare();
- st.execute(regionServer, regionServer);
- } catch (IOException e) {
- LOG.debug("Rollback started :"+ e.getMessage());
- st.rollback(regionServer, regionServer);
- }
- for (int i=0; !firstSplitCompleted && i<100; i++) {
- Thread.sleep(100);
- }
- assertTrue("fist split did not complete", firstSplitCompleted);
-
- RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
- Map<String, RegionState> rit = regionStates.getRegionsInTransition();
-
- for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) {
- Thread.sleep(100);
- }
- assertFalse("region still in transition", rit.containsKey(
- rit.containsKey(hri.getTable())));
-
- List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName);
- // Region server side split is successful.
- assertEquals("The parent region should be splitted", 2, onlineRegions.size());
- //Should be present in RIT
- List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager()
- .getRegionStates().getRegionsOfTable(tableName);
- // Master side should also reflect the same
- assertEquals("No of regions in master", 2, regionsOfTable.size());
- } finally {
- admin.setBalancerRunning(true, false);
- secondSplit = false;
- firstSplitCompleted = false;
- callRollBack = false;
- cluster.getMaster().setCatalogJanitorEnabled(true);
- TESTING_UTIL.deleteTable(tableName);
- }
- }
-
@Test(timeout = 60000)
public void testRITStateForRollback() throws Exception {
final TableName tableName =
@@ -382,20 +268,15 @@ public class TestSplitTransactionOnCluster {
/**
* A test that intentionally has master fail the processing of the split message.
- * Tests that the regionserver split ephemeral node gets cleaned up if it
- * crashes and that after we process server shutdown, the daughters are up on
- * line.
+ * Tests that after we process server shutdown, the daughters are up on line.
* @throws IOException
* @throws InterruptedException
- * @throws NodeExistsException
- * @throws KeeperException
- * @throws DeserializationException
+ * @throws ServiceException
*/
- @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling()
- throws IOException, InterruptedException, NodeExistsException, KeeperException,
- DeserializationException, ServiceException {
+ @Test (timeout = 300000) public void testRSSplitDaughtersAreOnlinedAfterShutdownHandling()
+ throws IOException, InterruptedException, ServiceException {
final byte [] tableName =
- Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling");
+ Bytes.toBytes("testRSSplitDaughtersAreOnlinedAfterShutdownHandling");
// Create table then get the single region for our new table.
HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
@@ -418,48 +299,15 @@ public class TestSplitTransactionOnCluster {
// Now, before we split, set special flag in master, a flag that has
// it FAIL the processing of split.
AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
- // Now try splitting and it should work.
- split(hri, server, regionCount);
-
- String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
- hri.getEncodedName());
- RegionTransition rt = null;
- Stat stats = null;
- List<HRegion> daughters = null;
- if (useZKForAssignment) {
- daughters = checkAndGetDaughters(tableName);
-
- // Wait till the znode moved to SPLIT
- for (int i=0; i<100; i++) {
- stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
- rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
- hri.getEncodedName()));
- if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
- Thread.sleep(100);
- }
- LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
- assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
- // Now crash the server, for ZK-less assignment, the server is auto aborted
- cluster.abortRegionServer(tableRegionIndex);
+ try {
+ // Now try splitting and it should work.
+ split(hri, server, regionCount);
+ } catch (RegionServerStoppedException rsse) {
+ // Expected. The regionserver should crash
}
+
waitUntilRegionServerDead();
awaitDaughters(tableName, 2);
- if (useZKForAssignment) {
- regions = cluster.getRegions(tableName);
- for (HRegion r: regions) {
- assertTrue(daughters.contains(r));
- }
-
- // Finally assert that the ephemeral SPLIT znode was cleaned up.
- for (int i=0; i<100; i++) {
- // wait a bit (10s max) for the node to disappear
- stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false);
- if (stats == null) break;
- Thread.sleep(100);
- }
- LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
- assertTrue(stats == null);
- }
} finally {
// Set this flag back.
AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
@@ -495,15 +343,8 @@ public class TestSplitTransactionOnCluster {
HRegionServer server = cluster.getRegionServer(tableRegionIndex);
printOutRegions(server, "Initial regions: ");
int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size();
- // Insert into zk a blocking znode, a znode of same name as region
- // so it gets in way of our splitting.
- ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
- if (useZKForAssignment) {
- ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
- hri, fakedServer);
- } else {
- regionStates.updateRegionState(hri, RegionState.State.CLOSING);
- }
+ regionStates.updateRegionState(hri, RegionState.State.CLOSING);
+
// Now try splitting.... should fail. And each should successfully
// rollback.
this.admin.split(hri.getRegionNameAsString());
@@ -515,13 +356,8 @@ public class TestSplitTransactionOnCluster {
assertEquals(regionCount, ProtobufUtil.getOnlineRegions(
server.getRSRpcServices()).size());
}
- if (useZKForAssignment) {
- // Now clear the zknode
- ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
- hri, fakedServer);
- } else {
- regionStates.regionOnline(hri, server.getServerName());
- }
+ regionStates.regionOnline(hri, server.getServerName());
+
// Now try splitting and it should work.
split(hri, server, regionCount);
// Get daughters
@@ -541,7 +377,7 @@ public class TestSplitTransactionOnCluster {
* @throws InterruptedException
*/
@Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()
- throws IOException, InterruptedException, ServiceException {
+ throws IOException, InterruptedException {
final byte [] tableName =
Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit");
@@ -698,103 +534,6 @@ public class TestSplitTransactionOnCluster {
}
/**
- * Verifies HBASE-5806. When splitting is partially done and the master goes down
- * when the SPLIT node is in either SPLIT or SPLITTING state.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws NodeExistsException
- * @throws KeeperException
- * @throws DeserializationException
- */
- @Test(timeout = 400000)
- public void testMasterRestartWhenSplittingIsPartial()
- throws IOException, InterruptedException, NodeExistsException,
- KeeperException, DeserializationException, ServiceException {
- final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
-
- if (!useZKForAssignment) {
- // This test doesn't apply if not using ZK for assignment
- return;
- }
-
- // Create table then get the single region for our new table.
- HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
- List<HRegion> regions = cluster.getRegions(tableName);
- HRegionInfo hri = getAndCheckSingleTableRegion(regions);
-
- int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
-
- // Turn off balancer so it doesn't cut in and mess up our placements.
- this.admin.setBalancerRunning(false, true);
- // Turn off the meta scanner so it don't remove parent on us.
- cluster.getMaster().setCatalogJanitorEnabled(false);
- ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(),
- "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable());
- try {
- // Add a bit of load up into the table so splittable.
- TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false);
- // Get region pre-split.
- HRegionServer server = cluster.getRegionServer(tableRegionIndex);
- printOutRegions(server, "Initial regions: ");
- // Now, before we split, set special flag in master, a flag that has
- // it FAIL the processing of split.
- AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
- // Now try splitting and it should work.
-
- this.admin.split(hri.getRegionNameAsString());
- checkAndGetDaughters(tableName);
- // Assert the ephemeral node is up in zk.
- String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
- Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
- LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
- + stats);
- byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName());
- RegionTransition rtd = RegionTransition.parseFrom(bytes);
- // State could be SPLIT or SPLITTING.
- assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)
- || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING));
-
- // abort and wait for new master.
- MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
-
- this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration());
-
- // Update the region to be offline and split, so that HRegionInfo#equals
- // returns true in checking rebuilt region states map.
- hri.setOffline(true);
- hri.setSplit(true);
- ServerName regionServerOfRegion = master.getAssignmentManager()
- .getRegionStates().getRegionServerOfRegion(hri);
- assertTrue(regionServerOfRegion != null);
-
- // Remove the block so that split can move ahead.
- AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
- String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
- Stat stat = new Stat();
- byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
- // ZKUtil.create
- for (int i=0; data != null && i<60; i++) {
- Thread.sleep(1000);
- data = ZKUtil.getDataNoWatch(zkw, node, stat);
- }
- assertNull("Waited too long for ZK node to be removed: "+node, data);
- RegionStates regionStates = master.getAssignmentManager().getRegionStates();
- assertTrue("Split parent should be in SPLIT state",
- regionStates.isRegionInState(hri, State.SPLIT));
- regionServerOfRegion = regionStates.getRegionServerOfRegion(hri);
- assertTrue(regionServerOfRegion == null);
- } finally {
- // Set this flag back.
- AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
- admin.setBalancerRunning(true, false);
- cluster.getMaster().setCatalogJanitorEnabled(true);
- t.close();
- zkw.close();
- }
- }
-
- /**
* Verifies HBASE-5806. Here the case is that splitting is completed but before the
* CJ could remove the parent region the master is killed and restarted.
* @throws IOException
@@ -830,20 +569,6 @@ public class TestSplitTransactionOnCluster {
this.admin.split(hri.getRegionNameAsString());
checkAndGetDaughters(tableName);
- // Assert the ephemeral node is up in zk.
- String path = ZKAssign.getNodeName(zkw, hri.getEncodedName());
- Stat stats = zkw.getRecoverableZooKeeper().exists(path, false);
- LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats="
- + stats);
- String node = ZKAssign.getNodeName(zkw, hri.getEncodedName());
- Stat stat = new Stat();
- byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat);
- // ZKUtil.create
- for (int i=0; data != null && i<60; i++) {
- Thread.sleep(1000);
- data = ZKUtil.getDataNoWatch(zkw, node, stat);
- }
- assertNull("Waited too long for ZK node to be removed: "+node, data);
MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster();
@@ -885,7 +610,6 @@ public class TestSplitTransactionOnCluster {
@Test(timeout = 60000)
public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception {
- ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
final TableName tableName =
TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent");
// Create table then get the single region for our new table.
@@ -916,11 +640,6 @@ public class TestSplitTransactionOnCluster {
tableName);
assertEquals("The specified table should present.", true, tableExists);
} finally {
- if (regions != null) {
- String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo()
- .getEncodedName());
- ZKUtil.deleteNodeFailSilent(zkw, node);
- }
admin.setBalancerRunning(true, false);
cluster.getMaster().setCatalogJanitorEnabled(true);
t.close();
@@ -929,7 +648,6 @@ public class TestSplitTransactionOnCluster {
@Test
public void testSplitWithRegionReplicas() throws Exception {
- ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
final TableName tableName =
TableName.valueOf("foobar");
HTableDescriptor htd = TESTING_UTIL.createTableDescriptor("foobar");
@@ -938,7 +656,6 @@ public class TestSplitTransactionOnCluster {
// Create table then get the single region for our new table.
HTable t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")},
TESTING_UTIL.getConfiguration());
- int count;
List<HRegion> oldRegions;
do {
oldRegions = cluster.getRegions(tableName);
@@ -960,9 +677,6 @@ public class TestSplitTransactionOnCluster {
regionServerIndex = cluster.getServerWith(region.getRegionName());
regionServer = cluster.getRegionServer(regionServerIndex);
assertTrue("not able to find a splittable region", region != null);
- String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
- region.getRegionInfo().getEncodedName());
- regionServer.getZooKeeper().sync(node);
SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2"));
try {
st.prepare();
@@ -1093,7 +807,7 @@ public class TestSplitTransactionOnCluster {
}
// We should not be able to assign it again
- am.assign(hri, true, true);
+ am.assign(hri, true);
assertFalse("Split region can't be assigned",
regionStates.isRegionInTransition(hri));
assertTrue(regionStates.isRegionInState(hri, State.SPLIT));
@@ -1176,17 +890,6 @@ public class TestSplitTransactionOnCluster {
throw new SplittingNodeCreationFailedException ();
}
};
- String node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
- region.getRegionInfo().getEncodedName());
- regionServer.getZooKeeper().sync(node);
- for (int i = 0; i < 100; i++) {
- // We expect the znode to be deleted by this time. Here the
- // znode could be in OPENED state and the
- // master has not yet deleted the znode.
- if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) {
- Thread.sleep(100);
- }
- }
try {
st.prepare();
st.execute(regionServer, regionServer);
@@ -1196,13 +899,7 @@ public class TestSplitTransactionOnCluster {
// This will at least make the test to fail;
assertTrue("Should be instance of CreateSplittingNodeFailedException",
e instanceof SplittingNodeCreationFailedException );
- node = ZKAssign.getNodeName(regionServer.getZooKeeper(),
- region.getRegionInfo().getEncodedName());
- {
- assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
- }
assertTrue(st.rollback(regionServer, regionServer));
- assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1);
}
} finally {
TESTING_UTIL.deleteTable(tableName);
@@ -1213,8 +910,6 @@ public class TestSplitTransactionOnCluster {
public void initialize(Server server, HRegion region) {
this.server = server;
this.watcher = server.getZooKeeper();
- splitTransactionCoordination = new MockedSplitTransactionCoordination(this, watcher, region);
- closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher);
}
}
@@ -1237,41 +932,7 @@ public class TestSplitTransactionOnCluster {
}
return super.rollback(server, services);
}
-
-
- }
-
- public static class MockedSplitTransactionCoordination extends ZKSplitTransactionCoordination {
-
- private HRegion currentRegion;
-
- public MockedSplitTransactionCoordination(CoordinatedStateManager coordinationProvider,
- ZooKeeperWatcher watcher, HRegion region) {
- super(coordinationProvider, watcher);
- currentRegion = region;
- }
-
- @Override
- public void completeSplitTransaction(RegionServerServices services, HRegion a, HRegion b,
- SplitTransactionDetails std, HRegion parent) throws IOException {
- if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
- .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
- try {
- if (!secondSplit){
- callRollBack = true;
- latch.await();
- }
- } catch (InterruptedException e) {
- }
-
}
- super.completeSplitTransaction(services, a, b, std, parent);
- if (this.currentRegion.getRegionInfo().getTable().getNameAsString()
- .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) {
- firstSplitCompleted = true;
- }
- }
- }
private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException {
for (int i = 0; i < 5; ++i) {
@@ -1313,22 +974,14 @@ public class TestSplitTransactionOnCluster {
private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
throws IOException, InterruptedException {
this.admin.split(hri.getRegionNameAsString());
- try {
- for (int i = 0; ProtobufUtil.getOnlineRegions(
- server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
- LOG.debug("Waiting on region to split");
- Thread.sleep(100);
- }
-
- assertFalse("Waited too long for split",
- ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
- } catch (RegionServerStoppedException e) {
- if (useZKForAssignment) {
- // If not using ZK for assignment, the exception may be expected.
- LOG.error(e);
- throw e;
- }
+ for (int i = 0; ProtobufUtil.getOnlineRegions(
+ server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
+ LOG.debug("Waiting on region to split");
+ Thread.sleep(100);
}
+
+ assertFalse("Waited too long for split",
+ ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
}
/**
@@ -1411,13 +1064,14 @@ public class TestSplitTransactionOnCluster {
private void waitUntilRegionServerDead() throws InterruptedException, InterruptedIOException {
// Wait until the master processes the RS shutdown
- for (int i=0; cluster.getMaster().getClusterStatus().
- getServers().size() > NB_SERVERS && i<100; i++) {
+ for (int i=0; (cluster.getMaster().getClusterStatus().getServers().size() > NB_SERVERS
+ || cluster.getLiveRegionServerThreads().size() > NB_SERVERS) && i<100; i++) {
LOG.info("Waiting on server to go down");
Thread.sleep(100);
}
- assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus().
- getServers().size() > NB_SERVERS);
+ assertFalse("Waited too long for RS to die",
+ cluster.getMaster().getClusterStatus(). getServers().size() > NB_SERVERS
+ || cluster.getLiveRegionServerThreads().size() > NB_SERVERS);
}
private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
deleted file mode 100644
index d9e8480..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Copyright The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.hadoop.hbase.LargeTests;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-/**
- * Like {@link TestRegionMergeTransaction} in that we're testing
- * {@link RegionMergeTransaction} only the below tests are against a running
- * cluster where {@link TestRegionMergeTransaction} is tests against bare
- * {@link HRegion}.
- */
-@Category(LargeTests.class)
-public class TestZKLessMergeOnCluster extends TestRegionMergeTransactionOnCluster {
- @BeforeClass
- public static void beforeAllTests() throws Exception {
- // Don't use ZK for region assignment
- TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false);
- setupOnce();
- }
-
- @AfterClass
- public static void afterAllTests() throws Exception {
- TestRegionMergeTransactionOnCluster.afterAllTests();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
deleted file mode 100644
index 1e3de64..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.hadoop.hbase.LargeTests;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-/**
- * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction}
- * only the below tests are against a running cluster where {@link TestSplitTransaction}
- * is tests against a bare {@link HRegion}.
- */
-@Category(LargeTests.class)
-public class TestZKLessSplitOnCluster extends TestSplitTransactionOnCluster {
- @BeforeClass
- public static void before() throws Exception {
- // Don't use ZK for region assignment
- TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false);
- setupOnce();
- }
-
- @AfterClass
- public static void after() throws Exception {
- TestSplitTransactionOnCluster.after();
- }
-}
-