You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/12/19 20:06:47 UTC
svn commit: r728112 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/
src/java/org/apache/hadoop/hbase/regionserver/
src/test/org/apache/hadoop/hbase/
src/test/org/apache/hadoop/hbase/regionserver/transactional/
Author: stack
Date: Fri Dec 19 11:06:46 2008
New Revision: 728112
URL: http://svn.apache.org/viewvc?rev=728112&view=rev
Log:
HBASE-1067 TestRegionRebalancing broken by running of hdfs shutdown thread
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Dec 19 11:06:46 2008
@@ -108,6 +108,8 @@
from org.apache.hadoop.hbase.DroppedSnapshotException
HBASE-1059 ConcurrentModificationException in notifyChangedReadersObservers
HBASE-1063 "File separator problem on Windows" (Max Lehn via Stack)
+ HBASE-1068 TestCompaction broken on hudson
+ HBASE-1067 TestRegionRebalancing broken by running of hdfs shutdown thread
IMPROVEMENTS
HBASE-901 Add a limit to key length, check key and value length on client side
@@ -177,7 +179,6 @@
HBASE-1065 Minor logging improvements in the master
HBASE-1053 bring recent rpc changes down from hadoop
HBASE-1056 [migration] enable blockcaching on .META. table
- HBASE-1068 TestCompaction broken on hudson
NEW FEATURES
HBASE-875 Use MurmurHash instead of JenkinsHash [in bloomfilters]
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Fri Dec 19 11:06:46 2008
@@ -34,6 +34,7 @@
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Threads;
/**
* This class creates a single process HBase cluster. One thread is created for
@@ -117,7 +118,8 @@
synchronized (regionThreads) {
HRegionServer server;
try {
- server = regionServerClass.getConstructor(HBaseConfiguration.class).newInstance(conf);
+ server = regionServerClass.getConstructor(HBaseConfiguration.class).
+ newInstance(conf);
} catch (Exception e) {
IOException ioe = new IOException();
ioe.initCause(e);
@@ -252,6 +254,17 @@
*/
public void shutdown() {
LOG.debug("Shutting down HBase Cluster");
+ // Be careful how the hdfs shutdown thread runs in context where more than
+ // one regionserver in the mix.
+ Thread shutdownThread = null;
+ synchronized (this.regionThreads) {
+ for (RegionServerThread t: this.regionThreads) {
+ Thread tt = t.getRegionServer().setHDFSShutdownThreadOnExit(null);
+ if (shutdownThread == null && tt != null) {
+ shutdownThread = tt;
+ }
+ }
+ }
if(this.master != null) {
this.master.shutdown();
}
@@ -280,6 +293,7 @@
}
}
}
+ Threads.shutdown(shutdownThread);
LOG.info("Shutdown " +
((this.regionThreads != null)? this.master.getName(): "0 masters") +
" " + this.regionThreads.size() + " region server(s)");
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri Dec 19 11:06:46 2008
@@ -521,17 +521,34 @@
}
join();
- LOG.info("Running hdfs shutdown thread");
- hdfsShutdownThread.start();
- try {
- hdfsShutdownThread.join();
- LOG.info("Hdfs shutdown thread completed.");
- } catch (InterruptedException e) {
- LOG.warn("hdfsShutdownThread.join() was interrupted", e);
- }
+ runThread(this.hdfsShutdownThread);
LOG.info(Thread.currentThread().getName() + " exiting");
}
+ /**
+ * Run and wait on passed thread in HRS context.
+ * @param t
+ */
+ public void runThread(final Thread t) {
+ if (t == null) {
+ return;
+ }
+ t.start();
+ Threads.shutdown(t);
+ }
+
+ /**
+ * Set the hdfs shutdown thread to run on exit. Pass null to disable
+ * running of the shutdown test. Needed by tests.
+ * @param t Thread to run. Pass null to disable tests.
+ * @return Previous occupant of the shutdown thread position.
+ */
+ public Thread setHDFSShutdownThreadOnExit(final Thread t) {
+ Thread old = this.hdfsShutdownThread;
+ this.hdfsShutdownThread = t;
+ return old;
+ }
+
/*
* Run init. Sets up hlog and starts up all server threads.
* @param c Extra configuration.
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Fri Dec 19 11:06:46 2008
@@ -1475,11 +1475,11 @@
// Returned array is sorted with the most recent addition last.
for(int i = maparray.length - 1;
i >= 0 && !hasEnoughVersions(versions, results); i--) {
- MapFile.Reader map = maparray[i];
- synchronized(map) {
+ MapFile.Reader r = maparray[i];
+ synchronized (r) {
// Do the priming read
ImmutableBytesWritable readval = new ImmutableBytesWritable();
- HStoreKey readkey = (HStoreKey)map.getClosest(key, readval);
+ HStoreKey readkey = (HStoreKey)r.getClosest(key, readval);
if (readkey == null) {
// map.getClosest returns null if the passed key is > than the
// last key in the map file. getClosest is a bit of a misnomer
@@ -1496,7 +1496,7 @@
break;
}
for (readval = new ImmutableBytesWritable();
- map.next(readkey, readval) && readkey.matchesRowCol(key);
+ r.next(readkey, readval) && readkey.matchesRowCol(key);
readval = new ImmutableBytesWritable()) {
if (get(readkey, readval.get(), versions, results, deletes, now)) {
break;
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java Fri Dec 19 11:06:46 2008
@@ -110,8 +110,28 @@
* @return the region server that was stopped
*/
public LocalHBaseCluster.RegionServerThread stopRegionServer(int serverNumber) {
- LocalHBaseCluster.RegionServerThread server = hbaseCluster.getRegionServers().get(serverNumber);
+ return stopRegionServer(serverNumber, true);
+ }
+
+ /**
+ * Shut down the specified region server cleanly
+ *
+ * @param serverNumber Used as index into a list.
+ * @param shutdownFS True is we are to shutdown the filesystem as part of this
+ * regionserver's shutdown. Usually we do but you do not want to do this if
+ * you are running multiple regionservers in a test and you shut down one
+ * before end of the test.
+ * @return the region server that was stopped
+ */
+ public LocalHBaseCluster.RegionServerThread stopRegionServer(int serverNumber,
+ final boolean shutdownFS) {
+ LocalHBaseCluster.RegionServerThread server =
+ hbaseCluster.getRegionServers().get(serverNumber);
LOG.info("Stopping " + server.toString());
+ if (!shutdownFS) {
+ // Stop the running of the hdfs shutdown thread in tests.
+ server.getRegionServer().setHDFSShutdownThreadOnExit(null);
+ }
server.getRegionServer().stop();
return server;
}
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestRegionRebalancing.java Fri Dec 19 11:06:46 2008
@@ -113,7 +113,7 @@
// kill a region server - total of 2
LOG.debug("Killing the 3rd region server.");
- cluster.stopRegionServer(2);
+ cluster.stopRegionServer(2, false);
cluster.waitOnRegionServer(2);
assertRegionsAreBalanced();
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java?rev=728112&r1=728111&r2=728112&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/transactional/DisabledTestHLogRecovery.java Fri Dec 19 11:06:46 2008
@@ -192,7 +192,7 @@
this.cluster.abortRegionServer(server);
} else {
- this.cluster.stopRegionServer(server);
+ this.cluster.stopRegionServer(server, false);
}
LOG.info(this.cluster.waitOnRegionServer(server) + " has been "
+ (abort ? "aborted" : "shut down"));