You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2013/08/15 22:10:48 UTC
svn commit: r1514464 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/util/
test/java/org/apache/hadoop/hbase/master/handler/
test/java/org/apache/hadoop/hbase/regionserver/
Author: jxiang
Date: Thu Aug 15 20:10:47 2013
New Revision: 1514464
URL: http://svn.apache.org/r1514464
Log:
HBASE-9207 An Offline SplitParent Region can be assigned breaking split references
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1514464&r1=1514463&r2=1514464&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Thu Aug 15 20:10:47 2013
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.master;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.lang.Thread.UncaughtExceptionHandler;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -1418,6 +1417,9 @@ public class AssignmentManager extends Z
region.getRegionNameAsString());
return;
}
+ if (isAssigningSplitParentRegion(region)) {
+ return;
+ }
RegionState state = addToRegionsInTransition(region,
hijack);
synchronized (state) {
@@ -1846,6 +1848,15 @@ public class AssignmentManager extends Z
}
}
+ private static boolean isAssigningSplitParentRegion(final HRegionInfo region) {
+ if (region.isSplitParent()) {
+ LOG.info("Skipping assign of " + region.getRegionNameAsString()
+ + ", already split, or still splitting");
+ return true;
+ }
+ return false;
+ }
+
private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) {
String tableName = region.getTableNameAsString();
boolean disabled = this.zkTable.isDisabledTable(tableName);
@@ -2527,28 +2538,6 @@ public class AssignmentManager extends Z
}
/**
- * Bulk user region assigner.
- * If failed assign, lets timeout in RIT do cleanup.
- */
- static class GeneralBulkAssigner extends StartupBulkAssigner {
- GeneralBulkAssigner(final Server server,
- final Map<ServerName, List<HRegionInfo>> bulkPlan,
- final AssignmentManager am) {
- super(server, bulkPlan, am);
- }
-
- @Override
- protected UncaughtExceptionHandler getUncaughtExceptionHandler() {
- return new UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- LOG.warn("Assigning regions in " + t.getName(), e);
- }
- };
- }
- }
-
- /**
* Manage bulk assigning to a server.
*/
static class SingleServerBulkAssigner implements Runnable {
@@ -2558,6 +2547,11 @@ public class AssignmentManager extends Z
SingleServerBulkAssigner(final ServerName regionserver,
final List<HRegionInfo> regions, final AssignmentManager am) {
+ for (Iterator<HRegionInfo> it = regions.iterator(); it.hasNext(); ) {
+ if (isAssigningSplitParentRegion(it.next())) {
+ it.remove();
+ }
+ }
this.regionserver = regionserver;
this.regions = regions;
this.assignmentManager = am;
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1514464&r1=1514463&r2=1514464&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Aug 15 20:10:47 2013
@@ -119,6 +119,7 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker;
import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
+import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
@@ -844,7 +845,7 @@ Server {
}
}
- void fixupDaughters(final MonitoredTask status) throws IOException {
+ void fixupDaughters(final MonitoredTask status) throws IOException, KeeperException {
final Map<HRegionInfo, Result> offlineSplitParents =
new HashMap<HRegionInfo, Result>();
// This visitor collects offline split parents in the .META. table
@@ -868,8 +869,12 @@ Server {
// Now work on our list of found parents. See if any we can clean up.
int fixups = 0;
for (Map.Entry<HRegionInfo, Result> e : offlineSplitParents.entrySet()) {
- fixups += ServerShutdownHandler.fixupDaughters(
+ String node = ZKAssign.getNodeName(zooKeeper, e.getKey().getEncodedName());
+ byte[] data = ZKUtil.getData(zooKeeper, node);
+ if (data == null) { // otherwise, splitting is still going on, skip it
+ fixups += ServerShutdownHandler.fixupDaughters(
e.getValue(), assignmentManager, catalogTracker);
+ }
}
if (fixups != 0) {
LOG.info("Scanned the catalog and fixed up " + fixups +
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java?rev=1514464&r1=1514463&r2=1514464&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java Thu Aug 15 20:10:47 2013
@@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.util;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
@@ -35,18 +34,11 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.backup.HFileArchiver;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.catalog.MetaEditor;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.regionserver.HRegion;
/**
@@ -157,20 +149,4 @@ public abstract class ModifyRegionUtils
});
return regionOpenAndInitThreadPool;
}
-
- /**
- * Trigger immediate assignment of the regions in round-robin fashion
- *
- * @param assignmentManager
- * @param regions
- */
- public static void assignRegions(final AssignmentManager assignmentManager,
- final List<HRegionInfo> regions) throws IOException {
- try {
- assignmentManager.assignUserRegionsToOnlineServers(regions);
- } catch (InterruptedException ie) {
- LOG.error("Caught " + ie + " during round-robin assignment");
- throw new InterruptedIOException(ie.getMessage());
- }
- }
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java?rev=1514464&r1=1514463&r2=1514464&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java Thu Aug 15 20:10:47 2013
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.master.handler;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -26,6 +27,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -40,16 +42,11 @@ import org.apache.hadoop.hbase.catalog.C
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterFileSystem;
-import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.ServerManager;
-import org.apache.hadoop.hbase.master.TestMaster;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.ZKTable;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -57,7 +54,6 @@ import org.junit.experimental.categories
public class TestCreateTableHandler {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class);
- private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler");
private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
private static boolean throwException = false;
@@ -73,10 +69,11 @@ public class TestCreateTableHandler {
}
@Test
- public void testCreateTableHandlerIfCalledTwoTimesAndFirstOneIsUnderProgress() throws Exception {
+ public void testCreateTableCalledTwiceAndFirstOneInProgress() throws Exception {
+ final byte[] tableName = Bytes.toBytes("testCreateTableCalledTwiceAndFirstOneInProgress");
final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
final HMaster m = cluster.getMaster();
- final HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+ final HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(FAMILYNAME));
final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null,
null) };
@@ -91,14 +88,47 @@ public class TestCreateTableHandler {
m.getCatalogTracker(), m.getAssignmentManager());
handler1.process();
for (int i = 0; i < 100; i++) {
- if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(TABLENAME)) {
+ if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) {
Thread.sleep(200);
}
}
- assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME));
+ assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName));
+ }
+ @Test (timeout=300000)
+ public void testCreateTableWithSplitRegion() throws Exception {
+ final byte[] tableName = Bytes.toBytes("testCreateTableWithSplitRegion");
+ final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ final HMaster m = cluster.getMaster();
+ final HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(new HColumnDescriptor(FAMILYNAME));
+ byte[] splitPoint = Bytes.toBytes("split-point");
+ long ts = System.currentTimeMillis();
+ HRegionInfo d1 = new HRegionInfo(tableName, null, splitPoint, false, ts);
+ HRegionInfo d2 = new HRegionInfo(tableName, splitPoint, null, false, ts + 1);
+ HRegionInfo parent = new HRegionInfo(tableName, null, null, true, ts + 2);
+ parent.setOffline(true);
+
+ Path tempdir = m.getMasterFileSystem().getTempDir();
+ FileSystem fs = m.getMasterFileSystem().getFileSystem();
+ Path tempTableDir = FSUtils.getTablePath(tempdir, tableName);
+ fs.delete(tempTableDir, true); // Clean up temp table dir if exists
+
+ final HRegionInfo[] hRegionInfos = new HRegionInfo[] {d1, d2, parent};
+ CreateTableHandler handler = new CreateTableHandler(m, m.getMasterFileSystem(),
+ m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos,
+ m.getCatalogTracker(), m.getAssignmentManager());
+ handler.process();
+ for (int i = 0; i < 200; i++) {
+ if (!TEST_UTIL.getHBaseAdmin().isTableAvailable(tableName)) {
+ Thread.sleep(300);
+ }
+ }
+ assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(tableName));
+ List<HRegionInfo> regions = m.getAssignmentManager().getRegionsOfTable(tableName);
+ assertFalse("Split parent should not be assigned", regions.contains(parent));
}
-
+
@Test (timeout=60000)
public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception {
byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated");
@@ -115,9 +145,8 @@ public class TestCreateTableHandler {
handler.process();
abortAndStartNewMaster(cluster);
assertTrue(cluster.getLiveMasterThreads().size() == 1);
-
}
-
+
private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException {
cluster.abortMaster(0);
cluster.waitOnMaster(0);
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java?rev=1514464&r1=1514463&r2=1514464&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java Thu Aug 15 20:10:47 2013
@@ -73,6 +73,7 @@ import org.junit.experimental.categories
* is tests against a bare {@link HRegion}.
*/
@Category(LargeTests.class)
+@SuppressWarnings("deprecation")
public class TestSplitTransactionOnCluster {
private static final Log LOG =
LogFactory.getLog(TestSplitTransactionOnCluster.class);
@@ -791,7 +792,6 @@ public class TestSplitTransactionOnClust
@Test(timeout = 180000)
public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception {
Configuration conf = TESTING_UTIL.getConfiguration();
- ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL);
String userTableName = "testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles";
HTableDescriptor htd = new HTableDescriptor(userTableName);
HColumnDescriptor hcd = new HColumnDescriptor("col");
@@ -1084,14 +1084,18 @@ public class TestSplitTransactionOnClust
LOG.debug("Customised master executed.");
}
}
-
+
private static class TransitionToSplittingFailedException extends IOException {
+ private static final long serialVersionUID = 7025885032995944524L;
+
public TransitionToSplittingFailedException() {
super();
}
}
private static class SplittingNodeCreationFailedException extends IOException {
+ private static final long serialVersionUID = 1652404976265623004L;
+
public SplittingNodeCreationFailedException () {
super();
}