You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2013/05/30 19:21:23 UTC
svn commit: r1487940 - in /hbase/branches/0.95:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-it/src/test/java/org/apache/hadoop/hbase/
hbase-server/src/main/java/org/apache/hadoop/hbase/master/
hbase-server/src/main/java/org/apache/...
Author: stack
Date: Thu May 30 17:21:22 2013
New Revision: 1487940
URL: http://svn.apache.org/r1487940
Log:
HBASE-8631 Meta Region First Recovery
Modified:
hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java
hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
Modified: hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java (original)
+++ hbase/branches/0.95/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java Thu May 30 17:21:22 2013
@@ -199,7 +199,8 @@ public abstract class ServerCallable<T>
// If the server is dead, we need to wait a little before retrying, to give
// a chance to the regions to be
- expectedSleep = ConnectionUtils.getPauseTime(pause, tries);
+ // tries hasn't been bumped up yet so we use "tries + 1" to get right pause time
+ expectedSleep = ConnectionUtils.getPauseTime(pause, tries + 1);
if (expectedSleep < MIN_WAIT_DEAD_SERVER
&& (location == null || getConnection().isDeadServer(location.getServerName()))) {
expectedSleep = ConnectionUtils.addJitter(MIN_WAIT_DEAD_SERVER, 0.10f);
Modified: hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java (original)
+++ hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java Thu May 30 17:21:22 2013
@@ -45,7 +45,7 @@ public abstract class IngestIntegrationT
protected void setUp(int numSlavesBase, Configuration conf) throws Exception {
tableName = this.getClass().getSimpleName();
- util = (conf == null) ? new IntegrationTestingUtility() : new IntegrationTestingUtility(conf);
+ util = getTestingUtil(conf);
LOG.info("Initializing cluster with " + numSlavesBase + " servers");
util.initializeCluster(numSlavesBase);
LOG.info("Done initializing cluster");
@@ -59,6 +59,17 @@ public abstract class IngestIntegrationT
Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret);
}
+ protected IntegrationTestingUtility getTestingUtil(Configuration conf) {
+ if (this.util == null) {
+ if (conf == null) {
+ this.util = new IntegrationTestingUtility();
+ } else {
+ this.util = new IntegrationTestingUtility(conf);
+ }
+ }
+ return util;
+ }
+
protected void setUp(int numSlavesBase) throws Exception {
setUp(numSlavesBase, null);
}
Modified: hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java (original)
+++ hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java Thu May 30 17:21:22 2013
@@ -33,7 +33,7 @@ import org.junit.experimental.categories
@Category(IntegrationTests.class)
public class IntegrationTestDataIngestWithChaosMonkey extends IngestIntegrationTestBase {
- private static final int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
+ private static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
// run for 5 min by default
private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000;
@@ -42,6 +42,11 @@ public class IntegrationTestDataIngestWi
@Before
public void setUp() throws Exception {
+ util= getTestingUtil(null);
+ if(!util.isDistributedCluster()) {
+ // In MiniCluster mode, we increase number of RS a little bit to speed the test
+ NUM_SLAVES_BASE = 5;
+ }
super.setUp(NUM_SLAVES_BASE);
monkey = new ChaosMonkey(util, ChaosMonkey.EVERY_MINUTE_RANDOM_ACTION_POLICY);
monkey.start();
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java Thu May 30 17:21:22 2013
@@ -93,13 +93,13 @@ public class MasterFileSystem {
final SplitLogManager splitLogManager;
private final MasterServices services;
- private final static PathFilter META_FILTER = new PathFilter() {
+ final static PathFilter META_FILTER = new PathFilter() {
public boolean accept(Path p) {
return HLogUtil.isMetaFile(p);
}
};
- private final static PathFilter NON_META_FILTER = new PathFilter() {
+ final static PathFilter NON_META_FILTER = new PathFilter() {
public boolean accept(Path p) {
return !HLogUtil.isMetaFile(p);
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java Thu May 30 17:21:22 2013
@@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -127,8 +128,8 @@ public class SplitLogManager extends Zoo
// When lastRecoveringNodeCreationTime is older than the following threshold, we'll check
// whether to GC stale recovering znodes
private long checkRecoveringTimeThreshold = 15000; // 15 seconds
- private final Set<ServerName> failedRecoveringRegionDeletions = Collections
- .synchronizedSet(new HashSet<ServerName>());
+ private final List<Pair<Set<ServerName>, Boolean>> failedRecoveringRegionDeletions = Collections
+ .synchronizedList(new ArrayList<Pair<Set<ServerName>, Boolean>>());
/**
* In distributedLogReplay mode, we need touch both splitlog and recovering-regions znodes in one
@@ -307,6 +308,7 @@ public class SplitLogManager extends Zoo
long t = EnvironmentEdgeManager.currentTimeMillis();
long totalSize = 0;
TaskBatch batch = new TaskBatch();
+ Boolean isMetaRecovery = (filter == null) ? null : false;
for (FileStatus lf : logfiles) {
// TODO If the log file is still being written to - which is most likely
// the case for the last log file - then its length will show up here
@@ -321,7 +323,12 @@ public class SplitLogManager extends Zoo
}
waitForSplittingCompletion(batch, status);
// remove recovering regions from ZK
- this.removeRecoveringRegionsFromZK(serverNames);
+ if (filter == MasterFileSystem.META_FILTER /* reference comparison */) {
+ // we split meta regions and user regions separately therefore logfiles are either all for
+ // meta or user regions but won't for both( we could have mixed situations in tests)
+ isMetaRecovery = true;
+ }
+ this.removeRecoveringRegionsFromZK(serverNames, isMetaRecovery);
if (batch.done != batch.installed) {
batch.isDead = true;
@@ -453,14 +460,18 @@ public class SplitLogManager extends Zoo
* It removes recovering regions under /hbase/recovering-regions/[encoded region name] so that the
* region server hosting the region can allow reads to the recovered region
* @param serverNames servers which are just recovered
+ * @param isMetaRecovery whether current recovery is for the meta region on
+ * <code>serverNames<code>
*/
- private void removeRecoveringRegionsFromZK(final Set<ServerName> serverNames) {
+ private void
+ removeRecoveringRegionsFromZK(final Set<ServerName> serverNames, Boolean isMetaRecovery) {
if (!this.distributedLogReplay) {
// the function is only used in WALEdit direct replay mode
return;
}
+ final String metaEncodeRegionName = HRegionInfo.FIRST_META_REGIONINFO.getEncodedName();
int count = 0;
Set<String> recoveredServerNameSet = new HashSet<String>();
if (serverNames != null) {
@@ -492,12 +503,20 @@ public class SplitLogManager extends Zoo
List<String> regions = ZKUtil.listChildrenNoWatch(watcher, watcher.recoveringRegionsZNode);
if (regions != null) {
for (String region : regions) {
+ if(isMetaRecovery != null) {
+ if ((isMetaRecovery && !region.equalsIgnoreCase(metaEncodeRegionName))
+ || (!isMetaRecovery && region.equalsIgnoreCase(metaEncodeRegionName))) {
+ // skip non-meta regions when recovering the meta region or
+ // skip the meta region when recovering user regions
+ continue;
+ }
+ }
String nodePath = ZKUtil.joinZNode(watcher.recoveringRegionsZNode, region);
List<String> failedServers = ZKUtil.listChildrenNoWatch(watcher, nodePath);
if (failedServers == null || failedServers.isEmpty()) {
ZKUtil.deleteNode(watcher, nodePath);
continue;
- }
+ }
if (recoveredServerNameSet.containsAll(failedServers)) {
ZKUtil.deleteNodeRecursively(watcher, nodePath);
} else {
@@ -514,7 +533,8 @@ public class SplitLogManager extends Zoo
} catch (KeeperException ke) {
LOG.warn("removeRecoveringRegionsFromZK got zookeeper exception. Will retry", ke);
if (serverNames != null && !serverNames.isEmpty()) {
- this.failedRecoveringRegionDeletions.addAll(serverNames);
+ this.failedRecoveringRegionDeletions.add(new Pair<Set<ServerName>, Boolean>(serverNames,
+ isMetaRecovery));
}
} finally {
this.recoveringRegionLock.unlock();
@@ -588,7 +608,7 @@ public class SplitLogManager extends Zoo
}
}
if (!needMoreRecovery) {
- ZKUtil.deleteNode(watcher, nodePath);
+ ZKUtil.deleteNodeRecursively(watcher, nodePath);
}
}
}
@@ -1384,12 +1404,16 @@ public class SplitLogManager extends Zoo
if (!failedRecoveringRegionDeletions.isEmpty()
|| (tot == 0 && tasks.size() == 0 && (timeInterval > checkRecoveringTimeThreshold))) {
// inside the function there have more checks before GC anything
- Set<ServerName> previouslyFailedDeletoins = null;
if (!failedRecoveringRegionDeletions.isEmpty()) {
- previouslyFailedDeletoins = new HashSet<ServerName>(failedRecoveringRegionDeletions);
- failedRecoveringRegionDeletions.removeAll(previouslyFailedDeletoins);
+ List<Pair<Set<ServerName>, Boolean>> previouslyFailedDeletions =
+ new ArrayList<Pair<Set<ServerName>, Boolean>>(failedRecoveringRegionDeletions);
+ failedRecoveringRegionDeletions.removeAll(previouslyFailedDeletions);
+ for (Pair<Set<ServerName>, Boolean> failedDeletion : previouslyFailedDeletions) {
+ removeRecoveringRegionsFromZK(failedDeletion.getFirst(), failedDeletion.getSecond());
+ }
+ } else {
+ removeRecoveringRegionsFromZK(null, null);
}
- removeRecoveringRegionsFromZK(previouslyFailedDeletoins);
}
}
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java Thu May 30 17:21:22 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.exception
import org.apache.hadoop.hbase.exceptions.NotServingRegionException;
import org.apache.hadoop.hbase.master.SplitLogManager;
import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
+import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -217,7 +218,14 @@ public class SplitLogWorker extends ZooK
this.watcher.splitLogZNode + " ... worker thread exiting.");
return;
}
- int offset = (int)(Math.random() * paths.size());
+ // pick meta wal firstly
+ int offset = (int) (Math.random() * paths.size());
+ for(int i = 0; i < paths.size(); i ++){
+ if(HLogUtil.isMetaFile(paths.get(i))) {
+ offset = i;
+ break;
+ }
+ }
for (int i = 0; i < paths.size(); i ++) {
int idx = (i + offset) % paths.size();
// don't call ZKSplitLog.getNodeName() because that will lead to
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java Thu May 30 17:21:22 2013
@@ -58,6 +58,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -1628,8 +1629,9 @@ public class HLogSplitter {
private final Set<String> recoveredRegions = Collections.synchronizedSet(new HashSet<String>());
private final Map<String, RegionServerWriter> writers =
new ConcurrentHashMap<String, RegionServerWriter>();
- // online encoded region name map
- private final Set<String> onlineRegions = Collections.synchronizedSet(new HashSet<String>());
+ // online encoded region name -> region location map
+ private final Map<String, HRegionLocation> onlineRegions =
+ new ConcurrentHashMap<String, HRegionLocation>();
private Map<byte[], HConnection> tableNameToHConnectionMap = Collections
.synchronizedMap(new TreeMap<byte[], HConnection>(Bytes.BYTES_COMPARATOR));
@@ -1648,8 +1650,17 @@ public class HLogSplitter {
private LogRecoveredEditsOutputSink logRecoveredEditsOutputSink;
private boolean hasEditsInDisablingOrDisabledTables = false;
+ private Configuration sinkConf;
public LogReplayOutputSink(int numWriters) {
super(numWriters);
+ // set a smaller retries to fast fail otherwise splitlogworker could be blocked for
+ // quite a while inside HConnection layer. The worker won't available for other
+ // tasks even after current task is preempted after a split task times out.
+ sinkConf = HBaseConfiguration.create(conf);
+ sinkConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+ HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER - 2);
+ sinkConf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT / 2);
+ sinkConf.setInt("hbase.client.serverside.retries.multiplier", 1);
this.waitRegionOnlineTimeOut = conf.getInt("hbase.splitlog.manager.timeout",
SplitLogManager.DEFAULT_TIMEOUT);
@@ -1763,7 +1774,8 @@ public class HLogSplitter {
}
try {
- loc = locateRegionAndRefreshLastFlushedSequenceId(hconn, table, kv.getRow());
+ loc = locateRegionAndRefreshLastFlushedSequenceId(hconn, table, kv.getRow(),
+ encodeRegionNameStr);
} catch (TableNotFoundException ex) {
// table has been deleted so skip edits of the table
LOG.info("Table " + Bytes.toString(table)
@@ -1848,22 +1860,22 @@ public class HLogSplitter {
* @throws IOException
*/
private HRegionLocation locateRegionAndRefreshLastFlushedSequenceId(HConnection hconn,
- byte[] table, byte[] row) throws IOException {
- HRegionLocation loc = hconn.getRegionLocation(table, row, false);
+ byte[] table, byte[] row, String originalEncodedRegionName) throws IOException {
+ HRegionLocation loc = onlineRegions.get(originalEncodedRegionName);
+ if(loc != null) return loc;
+
+ loc = hconn.getRegionLocation(table, row, false);
if (loc == null) {
throw new IOException("Can't locate location for row:" + Bytes.toString(row)
+ " of table:" + Bytes.toString(table));
}
- if (onlineRegions.contains(loc.getRegionInfo().getEncodedName())) {
- return loc;
- }
Long lastFlushedSequenceId = -1l;
loc = waitUntilRegionOnline(loc, row, this.waitRegionOnlineTimeOut);
Long cachedLastFlushedSequenceId = lastFlushedSequenceIds.get(loc.getRegionInfo()
.getEncodedName());
- onlineRegions.add(loc.getRegionInfo().getEncodedName());
+ onlineRegions.put(loc.getRegionInfo().getEncodedName(), loc);
// retrieve last flushed sequence Id from ZK. Because region postOpenDeployTasks will
// update the value for the region
RegionStoreSequenceIds ids =
@@ -2116,7 +2128,7 @@ public class HLogSplitter {
synchronized (this.tableNameToHConnectionMap) {
hconn = this.tableNameToHConnectionMap.get(tableName);
if (hconn == null) {
- hconn = HConnectionManager.createConnection(conf);
+ hconn = HConnectionManager.createConnection(sinkConf);
this.tableNameToHConnectionMap.put(tableName, hconn);
}
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java Thu May 30 17:21:22 2013
@@ -246,7 +246,11 @@ public class HLogUtil {
}
public static boolean isMetaFile(Path p) {
- if (p.getName().endsWith(HLog.META_HLOG_FILE_EXTN)) {
+ return isMetaFile(p.getName());
+ }
+
+ public static boolean isMetaFile(String p) {
+ if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
return true;
}
return false;
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java?rev=1487940&r1=1487939&r2=1487940&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java Thu May 30 17:21:22 2013
@@ -26,6 +26,7 @@ import static org.apache.hadoop.hbase.Sp
import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_task_err;
import static org.apache.hadoop.hbase.SplitLogCounters.tot_wkr_task_resigned;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -995,6 +996,72 @@ public class TestDistributedLogSplitting
}
}
+ @Test(timeout = 300000)
+ public void testMetaRecoveryInZK() throws Exception {
+ LOG.info("testMetaRecoveryInZK");
+ Configuration curConf = HBaseConfiguration.create();
+ curConf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
+ startCluster(NUM_RS, curConf);
+
+ // turn off load balancing to prevent regions from moving around otherwise
+ // they will consume recovered.edits
+ master.balanceSwitch(false);
+ FileSystem fs = master.getMasterFileSystem().getFileSystem();
+ final ZooKeeperWatcher zkw = new ZooKeeperWatcher(curConf, "table-creation", null);
+ List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+
+ installTable(zkw, "table", "family", 40);
+ List<HRegionInfo> regions = null;
+ HRegionServer hrs = null;
+ for (int i = 0; i < NUM_RS; i++) {
+ boolean isCarryingMeta = false;
+ hrs = rsts.get(i).getRegionServer();
+ regions = ProtobufUtil.getOnlineRegions(hrs);
+ for (HRegionInfo region : regions) {
+ if (region.isMetaRegion()) {
+ isCarryingMeta = true;
+ break;
+ }
+ }
+ if (!isCarryingMeta) {
+ continue;
+ }
+ break;
+ }
+
+ LOG.info("#regions = " + regions.size());
+ Set<HRegionInfo> tmpRegions = new HashSet<HRegionInfo>();
+ tmpRegions.add(HRegionInfo.FIRST_META_REGIONINFO);
+ master.getMasterFileSystem().prepareMetaLogReplay(hrs.getServerName(), tmpRegions);
+ Set<ServerName> failedServers = new HashSet<ServerName>();
+ failedServers.add(hrs.getServerName());
+ master.getMasterFileSystem().prepareLogReplay(failedServers);
+ boolean isMetaRegionInRecovery = false;
+ List<String> recoveringRegions =
+ zkw.getRecoverableZooKeeper().getChildren(zkw.recoveringRegionsZNode, false);
+ for (String curEncodedRegionName : recoveringRegions) {
+ if (curEncodedRegionName.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
+ isMetaRegionInRecovery = true;
+ break;
+ }
+ }
+ assertTrue(isMetaRegionInRecovery);
+
+ master.getMasterFileSystem().splitMetaLog(hrs.getServerName());
+
+ isMetaRegionInRecovery = false;
+ recoveringRegions =
+ zkw.getRecoverableZooKeeper().getChildren(zkw.recoveringRegionsZNode, false);
+ for (String curEncodedRegionName : recoveringRegions) {
+ if (curEncodedRegionName.equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
+ isMetaRegionInRecovery = true;
+ break;
+ }
+ }
+ // meta region should be recovered
+ assertFalse(isMetaRegionInRecovery);
+ }
+
HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception {
return installTable(zkw, tname, fname, nrs, 0);
}