You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jd...@apache.org on 2010/04/06 19:46:30 UTC
svn commit: r931241 - in /hadoop/hbase/branches/0.20_pre_durability: ./ lib/
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/mapred/
src/java/org/apache/hadoop/hbase/mapreduce/ src/java/org/ap...
Author: jdcryans
Date: Tue Apr 6 17:46:29 2010
New Revision: 931241
URL: http://svn.apache.org/viewvc?rev=931241&view=rev
Log:
Adding the following jiras that were added after rev 919707 and that are needed:
HBASE-2034 Client sync block can cause 1 thread of a multi-threaded client
to block all others (Karthik Ranganathan via Stack)
HBASE-2355 Unsynchronized logWriters map is mutated from several threads in
HLog splitting (Todd Lipcon via Andrew Purtell)
HBASE-2358 Store doReconstructionLog will fail if oldlogfile.log is
empty and won't load region (Cosmin Lehene via Stack)
HBASE-2365 Double-assignment around split
HBASE-2174 Stop from resolving HRegionServer addresses to names using DNS on
every heartbeat (Karthik Ranganathan via Stack)
HBASE-2087 The wait on compaction because "Too many store files"
holds up all flushing
HBASE-2252 Mapping a very big table kills region servers
HBASE-2277 Update branch to hadoop 0.20.2
Added:
hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar (with props)
hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar (with props)
Removed:
hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.1-hdfs127-core.jar
hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.1-test.jar
Modified:
hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java
Modified: hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20_pre_durability/CHANGES.txt Tue Apr 6 17:46:29 2010
@@ -28,6 +28,13 @@ Release 0.20.4 - Unreleased
half-written hfile (Ruslan Salyakhov via Stack)
HBASE-2288 Shell fails on alter
HBASE-2293 CME in RegionManager#isMetaServer
+ HBASE-2034 Client sync block can cause 1 thread of a multi-threaded client
+ to block all others (Karthik Ranganathan via Stack)
+ HBASE-2355 Unsynchronized logWriters map is mutated from several threads in
+ HLog splitting (Todd Lipcon via Andrew Purtell)
+ HBASE-2358 Store doReconstructionLog will fail if oldlogfile.log is
+ empty and won't load region (Cosmin Lehene via Stack)
+ HBASE-2365 Double-assignment around split
IMPROVEMENTS
HBASE-2180 Bad read performance from synchronizing hfile.fddatainputstream
@@ -42,6 +49,12 @@ Release 0.20.4 - Unreleased
of the supplied byte array (Bruno Dumon via Stack)
HBASE-2241 Change balancer sloppyness from 0.1 to 0.3
HBASE-2262 ZKW.ensureExists should check for existence
+ HBASE-2174 Stop from resolving HRegionServer addresses to names using DNS on
+ every heartbeat (Karthik Ranganathan via Stack)
+ HBASE-2087 The wait on compaction because "Too many store files"
+ holds up all flushing
+ HBASE-2252 Mapping a very big table kills region servers
+ HBASE-2277 Update branch to hadoop 0.20.2
NEW FEATURES
HBASE-2257 [stargate] multiuser mode
Added: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar?rev=931241&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-core.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar?rev=931241&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/hbase/branches/0.20_pre_durability/lib/hadoop-0.20.2-test.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/HServerInfo.java Tue Apr 6 17:46:29 2010
@@ -137,8 +137,15 @@ public class HServerInfo implements Writ
*/
public synchronized String getServerName() {
if (this.serverName == null) {
+ // if we have the hostname of the RS, use it
+ if(this.name != null) {
+ this.serverName = getServerName(this.name, this.serverAddress.getPort(), this.startCode);
+ }
+ // go to DNS name resolution only if we dont have the name of the RS
+ else {
this.serverName = getServerName(this.serverAddress, this.startCode);
}
+ }
return this.serverName;
}
@@ -219,7 +226,7 @@ public class HServerInfo implements Writ
* @param info
* @return the server name in the form hostname_startcode_port
*/
- public static String getServerName(HServerInfo info) {
+ private static String getServerName(HServerInfo info) {
return getServerName(info.getServerAddress(), info.getStartCode());
}
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Tue Apr 6 17:46:29 2010
@@ -627,16 +627,12 @@ public class HConnectionManager implemen
return this.rootRegionLocation;
}
} else if (Bytes.equals(tableName, META_TABLE_NAME)) {
- synchronized (metaRegionLock) {
- // This block guards against two threads trying to load the meta
- // region at the same time. The first will load the meta region and
- // the second will use the value that the first one found.
- return locateRegionInMeta(ROOT_TABLE_NAME, tableName, row, useCache);
- }
+ return locateRegionInMeta(ROOT_TABLE_NAME, tableName, row, useCache,
+ metaRegionLock);
} else {
- synchronized(userRegionLock){
- return locateRegionInMeta(META_TABLE_NAME, tableName, row, useCache);
- }
+ // Region not in the cache - have to go to the meta RS
+ return locateRegionInMeta(META_TABLE_NAME, tableName, row, useCache,
+ userRegionLock);
}
}
@@ -645,18 +641,17 @@ public class HConnectionManager implemen
* info that contains the table and row we're seeking.
*/
private HRegionLocation locateRegionInMeta(final byte [] parentTable,
- final byte [] tableName, final byte [] row, boolean useCache)
+ final byte [] tableName, final byte [] row, boolean useCache,
+ Object regionLockObject)
throws IOException {
HRegionLocation location = null;
- // If supposed to be using the cache, then check it for a possible hit.
- // Otherwise, delete any existing cached location so it won't interfere.
+ // If we are supposed to be using the cache, look in the cache to see if
+ // we already have the region.
if (useCache) {
location = getCachedLocation(tableName, row);
if (location != null) {
return location;
}
- } else {
- deleteCachedLocation(tableName, row);
}
// build the key of the meta region we should be looking for.
@@ -676,10 +671,29 @@ public class HConnectionManager implemen
HRegionInterface server =
getHRegionConnection(metaLocation.getServerAddress());
+ Result regionInfoRow = null;
+ // This block guards against two threads trying to load the meta
+ // region at the same time. The first will load the meta region and
+ // the second will use the value that the first one found.
+ synchronized(regionLockObject) {
+ // Check the cache again for a hit in case some other thread made the
+ // same query while we were waiting on the lock. If not supposed to
+ // be using the cache, delete any existing cached location so it won't
+ // interfere.
+ if (useCache) {
+ location = getCachedLocation(tableName, row);
+ if (location != null) {
+ return location;
+ }
+ } else {
+ deleteCachedLocation(tableName, row);
+ }
+
// Query the root or meta region for the location of the meta region
- Result regionInfoRow = server.getClosestRowBefore(
+ regionInfoRow = server.getClosestRowBefore(
metaLocation.getRegionInfo().getRegionName(), metaKey,
HConstants.CATALOG_FAMILY);
+ }
if (regionInfoRow == null) {
throw new TableNotFoundException(Bytes.toString(tableName));
}
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java Tue Apr 6 17:46:29 2010
@@ -110,6 +110,7 @@ implements InputFormat<ImmutableBytesWri
Scan scan = new Scan(firstRow, endRow);
scan.addColumns(trrInputColumns);
scan.setOldFilter(trrRowFilter);
+ scan.setCacheBlocks(false);
this.scanner = this.htable.getScanner(scan);
} else {
LOG.debug("TIFB.restart, firstRow: " +
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java Tue Apr 6 17:46:29 2010
@@ -119,13 +119,12 @@ implements Configurable {
scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
}
- if (conf.get(SCAN_CACHEBLOCKS) != null) {
- scan.setCacheBlocks(Boolean.parseBoolean(conf.get(SCAN_CACHEBLOCKS)));
- }
-
if (conf.get(SCAN_CACHEDROWS) != null) {
scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
}
+
+ // false by default, full table scans generate too much BC churn
+ scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
}
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java Tue Apr 6 17:46:29 2010
@@ -53,7 +53,7 @@ class ProcessRegionOpen extends ProcessR
@Override
public String toString() {
- return "PendingOpenOperation from " + HServerInfo.getServerName(serverInfo);
+ return "PendingOpenOperation from " + serverInfo.getServerName();
}
@Override
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java Tue Apr 6 17:46:29 2010
@@ -74,7 +74,7 @@ class ProcessServerShutdown extends Regi
*/
public ProcessServerShutdown(HMaster master, HServerInfo serverInfo) {
super(master);
- this.deadServer = HServerInfo.getServerName(serverInfo);
+ this.deadServer = serverInfo.getServerName();
this.deadServerAddress = serverInfo.getServerAddress();
this.logSplit = false;
this.rootRescanned = false;
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/RegionManager.java Tue Apr 6 17:46:29 2010
@@ -980,7 +980,7 @@ class RegionManager implements HConstant
regionsInTransition.put(info.getRegionNameAsString(), s);
}
}
- if (force || (!s.isPendingOpen() || !s.isOpen())) {
+ if (force || (!s.isPendingOpen() && !s.isOpen())) {
s.setUnassigned();
}
}
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/master/ServerManager.java Tue Apr 6 17:46:29 2010
@@ -155,7 +155,7 @@ class ServerManager implements HConstant
public void regionServerStartup(final HServerInfo serverInfo)
throws Leases.LeaseStillHeldException {
HServerInfo info = new HServerInfo(serverInfo);
- String serverName = HServerInfo.getServerName(info);
+ String serverName = info.getServerName();
if (serversToServerInfo.containsKey(serverName) ||
deadServers.contains(serverName)) {
LOG.debug("Server start was rejected: " + serverInfo);
@@ -212,7 +212,7 @@ class ServerManager implements HConstant
*/
public void recordNewServer(HServerInfo info, boolean useInfoLoad) {
HServerLoad load = useInfoLoad ? info.getLoad() : new HServerLoad();
- String serverName = HServerInfo.getServerName(info);
+ String serverName = info.getServerName();
info.setLoad(load);
// We must set this watcher here because it can be set on a fresh start
// or on a failover
@@ -834,7 +834,7 @@ class ServerManager implements HConstant
serverAddressToServerInfo.remove(serverAddress);
HServerInfo info = serversToServerInfo.remove(server);
if (info != null) {
- String serverName = HServerInfo.getServerName(info);
+ String serverName = info.getServerName();
HServerLoad load = serversToLoad.remove(serverName);
if (load != null) {
synchronized (loadToServers) {
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Tue Apr 6 17:46:29 2010
@@ -923,7 +923,8 @@ public class HLog implements HConstants,
final HBaseConfiguration conf)
throws IOException {
final Map<byte [], WriterAndPath> logWriters =
- new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR);
+ Collections.synchronizedMap(
+ new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR));
List<Path> splits = null;
// Number of threads to use when log splitting to rewrite the logs.
@@ -1177,7 +1178,7 @@ public class HLog implements HConstants,
* @return the HLog directory name
*/
public static String getHLogDirectoryName(HServerInfo info) {
- return getHLogDirectoryName(HServerInfo.getServerName(info));
+ return getHLogDirectoryName(info.getServerName());
}
/*
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Tue Apr 6 17:46:29 2010
@@ -222,7 +222,25 @@ class MemStoreFlusher extends Thread imp
* not flushed.
*/
private boolean flushRegion(HRegion region, boolean removeFromQueue) {
- checkStoreFileCount(region);
+ // if removeFromQueue, then we come from flushSomeRegions and we need
+ // to block if there's too many store files. Else, we don't want to hang
+ // the main flushing thread so we'll just the region at the end of the
+ // queue if there's too many files.
+ if (removeFromQueue) {
+ checkStoreFileCount(region);
+ } else if (isTooManyStoreFiles(region)) {
+ LOG.warn("Region " + region.getRegionNameAsString() + " has too many " +
+ "store files, putting it back at the end of the flush queue.");
+ server.compactSplitThread.compactionRequested(region, getName());
+ // If there's only this item in the queue or they are all in this
+ // situation, we will loop at lot. Sleep a bit.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) { } // just continue
+ flushQueue.add(region);
+ // Tell a lie, it's not flushed but it's ok
+ return true;
+ }
synchronized (regionsInQueue) {
// See comment above for removeFromQueue on why we do not
// take the region out of the set. If removeFromQueue is true, remove it
@@ -310,6 +328,15 @@ class MemStoreFlusher extends Thread imp
}
}
+ private boolean isTooManyStoreFiles(HRegion region) {
+ for (Store hstore: region.stores.values()) {
+ if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Check if the regionserver's memstore memory usage is greater than the
* limit. If so, flush regions with the biggest memstores until we're down
Modified: hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=931241&r1=931240&r2=931241&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hadoop/hbase/branches/0.20_pre_durability/src/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Apr 6 17:46:29 2010
@@ -302,11 +302,11 @@ public class Store implements HConstants
// Nothing to do.
return -1;
}
- // Check its not empty.
- FileStatus [] stats = this.fs.listStatus(reconstructionLog);
- if (stats == null || stats.length == 0) {
+ FileStatus stat = this.fs.getFileStatus(reconstructionLog);
+ if (stat.getLen() <= 0) {
LOG.warn("Passed reconstruction log " + reconstructionLog +
- " is zero-length");
+ " is zero-length. Deleting existing file");
+ fs.delete(reconstructionLog, false);
return -1;
}
// TODO: This could grow large and blow heap out. Need to get it into