You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/04/03 04:04:27 UTC
svn commit: r1308651 [2/2] - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/wal/ main/java/org/...
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestLogSplitOnMasterFailover.java Tue Apr 3 02:04:26 2012
@@ -65,7 +65,7 @@ public class TestLogSplitOnMasterFailove
private static final int NUM_MASTERS = 2;
private static final int NUM_RS = 2;
- private static final int NUM_ROWS = 8000;
+ private static final int NUM_ROWS = 6000;
private static final int COLS_PER_ROW = 30;
private static final byte[] TABLE_BYTES = Bytes.toBytes("myTable");
@@ -73,6 +73,8 @@ public class TestLogSplitOnMasterFailove
private static Compression.Algorithm COMPRESSION = Compression.Algorithm.GZ;
+ private Set<String> logsSplitByNewMaster = new HashSet<String>();
+
/**
* A worker that inserts data into HBase on a separate thread. This is the
* reusable base class, which is subclassed for use in this particular test.
@@ -88,12 +90,17 @@ public class TestLogSplitOnMasterFailove
private Semaphore halfRowsLoaded = new Semaphore(0);
private Semaphore dataLoadVerifyFinished = new Semaphore(0);
+ private Semaphore newMasterAssignedRegions = new Semaphore(0);
private final Configuration conf;
+ private final HBaseTestingUtility testUtil;
+
private volatile Thread myThread;
+ private volatile int numUserRegions;
- public DataLoader(Configuration conf) {
+ public DataLoader(Configuration conf, HBaseTestingUtility testUtil) {
this.conf = conf;
+ this.testUtil = testUtil;
}
@Override
@@ -101,7 +108,7 @@ public class TestLogSplitOnMasterFailove
myThread = Thread.currentThread();
myThread.setName(getClass().getSimpleName());
try {
- HBaseTestingUtility.createPreSplitLoadTestTable(conf,
+ numUserRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf,
TABLE_BYTES, CF_BYTES, COMPRESSION, DataBlockEncoding.NONE);
t = new HTable(conf, TABLE_BYTES);
@@ -122,7 +129,7 @@ public class TestLogSplitOnMasterFailove
}
}
- private void loadData() throws IOException {
+ private void loadData() throws IOException, InterruptedException {
Random rand = new Random(190879817L);
int bytesInserted = 0;
for (int i = 0; i < NUM_ROWS; ++i) {
@@ -153,6 +160,8 @@ public class TestLogSplitOnMasterFailove
LOG.info("Loaded half of the rows (" + rowsLoaded
+ "), waking up main thread");
halfRowsLoaded.release();
+ newMasterAssignedRegions.acquire();
+ LOG.info("All regions assigned, proceeding with load test");
}
}
LOG.info("Approximate number of bytes inserted: " + bytesInserted);
@@ -193,6 +202,11 @@ public class TestLogSplitOnMasterFailove
LOG.debug("Waiting until half of the rows are loaded");
halfRowsLoaded.acquire();
}
+
+ public void notifyThatNewMasterAssignedRegions() {
+ newMasterAssignedRegions.release();
+ }
+
public void requestShutdown() {
shutdownRequested = true;
}
@@ -209,12 +223,21 @@ public class TestLogSplitOnMasterFailove
public void join() throws InterruptedException {
myThread.join();
}
+
+ public void waitUntilRegionsAssigned() throws IOException{
+ header("Waiting until all " + numUserRegions
+ + " regions are online with new master");
+ testUtil.waitUntilAllRegionsAssigned(numUserRegions);
+ // Tell the load test to proceed.
+ notifyThatNewMasterAssignedRegions();
+ }
}
@Test(timeout=180000)
public void testWithRegularLogSplitting() throws Exception {
ZooKeeperWrapper.setNamespaceForTesting();
conf.setBoolean(HConstants.DISTRIBUTED_LOG_SPLITTING_KEY, false);
+ conf.setInt(HConstants.ZOOKEEPER_SESSION_TIMEOUT, 30000);
runTest();
}
@@ -235,8 +258,7 @@ public class TestLogSplitOnMasterFailove
List<HMaster> masters = miniCluster().getMasters();
header("Starting data loader");
- DataLoader dataLoader =
- new DataLoader(conf);
+ DataLoader dataLoader = new DataLoader(conf, TEST_UTIL);
Thread inserterThread = new Thread(dataLoader);
inserterThread.start();
dataLoader.waitUntilHalfRowsLoaded();
@@ -253,9 +275,16 @@ public class TestLogSplitOnMasterFailove
miniCluster().getRegionServer(i).getServerInfo().getServerName());
}
rsToKill.kill();
- // Wait until the regionserver actually goes down.
- while (miniCluster().getLiveRegionServerThreads().size() == NUM_RS) {
- Threads.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
+
+ // Wait until the regionserver actually goes down. Furthermore, to make
+ // things more interesting, wait until the active master starts splitting
+ // the logs before we kill it. We have to do this in a tight polling loop
+ // because with distributed log splitting the active master might actually
+ // be able to split the dead regionserver's logs really quickly.
+ HMaster activeMaster = miniCluster().getMaster(activeIndex);
+ while (activeMaster.getNumDeadServerLogSplitRequests() == 0 ||
+ miniCluster().getLiveRegionServerThreads().size() == NUM_RS) {
+ Threads.sleepWithoutInterrupt(10);
}
// Check that we have some logs.
@@ -283,6 +312,8 @@ public class TestLogSplitOnMasterFailove
HMaster master = masters.get(0);
assertTrue(master.isActiveMaster());
+ dataLoader.waitUntilRegionsAssigned();
+
dataLoader.waitUntilFinishedOrFailed();
dataLoader.join();
dataLoader.assertSuccess();
@@ -291,16 +322,22 @@ public class TestLogSplitOnMasterFailove
List<String> logDirsSplitAtStartup = master.getLogDirsSplitOnStartup();
LOG.info("Log dirs split at startup: " + logDirsSplitAtStartup);
- Set<String> logsSplit = new HashSet<String>();
- logsSplit.addAll(logDirsSplitAtStartup);
- String logDirToBeSplit = killedRsName + "-splitting";
+ logsSplitByNewMaster.addAll(logDirsSplitAtStartup);
+ String logDirToBeSplit = killedRsName + HConstants.HLOG_SPLITTING_EXT;
assertTrue("Log directory " + logDirToBeSplit + " was not split " +
"on startup. Logs split: " + logDirsSplitAtStartup,
- logsSplit.contains(logDirToBeSplit));
- for (String logNotToSplit : otherRsNames) {
- assertFalse("Log directory " + logNotToSplit
- + " should not have been split", logsSplit.contains(logNotToSplit));
+ logWasSplit(logDirToBeSplit));
+ for (String logDirNotToSplit : otherRsNames) {
+ assertFalse("Log directory " + logDirNotToSplit
+ + " should not have been split: " + logDirsSplitAtStartup,
+ logWasSplit(logDirNotToSplit));
}
}
+ private boolean logWasSplit(String rsName) {
+ return logsSplitByNewMaster.contains(rsName)
+ || logsSplitByNewMaster.contains(rsName
+ + HConstants.HLOG_SPLITTING_EXT);
+ }
+
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java Tue Apr 3 02:04:26 2012
@@ -71,6 +71,8 @@ public class TestMasterFailover extends
killActiveMasterAndWaitToStop();
assertEquals(1, masters.size());
+
+ waitForActiveMasterAndVerify();
}
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java Tue Apr 3 02:04:26 2012
@@ -31,6 +31,7 @@ import org.junit.Test;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.StoppableImpl;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
@@ -82,7 +83,7 @@ public class TestOldLogsCleaner {
String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8");
FileSystem fs = FileSystem.get(c);
- AtomicBoolean stop = new AtomicBoolean(false);
+ StoppableImpl stop = new StoppableImpl();
OldLogsCleaner cleaner = new OldLogsCleaner(1000, stop,c, fs, oldLogDir);
// Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java Tue Apr 3 02:04:26 2012
@@ -20,10 +20,8 @@
package org.apache.hadoop.hbase.master;
import static org.junit.Assert.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.StopStatus;
import org.apache.hadoop.hbase.master.RegionServerOperationQueue.ProcessingResultCode;
import org.junit.After;
import org.junit.Before;
@@ -37,13 +35,17 @@ import org.junit.Test;
public class TestRegionServerOperationQueue {
private RegionServerOperationQueue queue;
private Configuration conf;
- private AtomicBoolean closed;
+ private StopStatus stopStatus = new StopStatus() {
+ @Override
+ public boolean isStopped() {
+ return false;
+ }
+ };
@Before
public void setUp() throws Exception {
- this.closed = new AtomicBoolean(false);
this.conf = new Configuration();
- this.queue = new RegionServerOperationQueue(this.conf, this.closed);
+ this.queue = new RegionServerOperationQueue(this.conf, stopStatus);
}
@After
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionStateOnMasterFailure.java Tue Apr 3 02:04:26 2012
@@ -338,7 +338,7 @@ public class TestRegionStateOnMasterFail
logMsg("Killing master right before it can process the event "
+ eventType + " for region " + openedRegion);
HBaseEventHandler.unregisterListener(this);
- localCluster().getActiveMaster().killMaster();
+ miniCluster().killActiveMaster();
terminateEventThread = true;
} else {
logMsg("Skipping event for region " + openedRegion
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java?rev=1308651&r1=1308650&r2=1308651&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java Tue Apr 3 02:04:26 2012
@@ -25,13 +25,13 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.master.SplitLogManager.Task;
import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
@@ -68,7 +68,24 @@ public class TestSplitLogManager {
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
- static AtomicBoolean stopper = new AtomicBoolean();
+ private volatile boolean stopped = false;
+
+ private Stoppable stopper = new Stoppable() {
+ @Override
+ public boolean isStopped() {
+ return stopped;
+ }
+
+ @Override
+ public void stop(String why) {
+ stopped = true;
+ }
+
+ @Override
+ public String getStopReason() {
+ return "test";
+ }
+ };
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -82,7 +99,6 @@ public class TestSplitLogManager {
@Before
public void setup() throws Exception {
- stopper.set(false);
conf = TEST_UTIL.getConfiguration();
zkw = ZooKeeperWrapper.createInstance(conf, "split-log-manager-tests");
zkw.deleteChildrenRecursively(zkw.parentZNode);
@@ -98,7 +114,7 @@ public class TestSplitLogManager {
@After
public void teardown() throws IOException, KeeperException {
- stopper.set(true);
+ stopped = true;
zkw.close();
slm.stop();
}