You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/05/01 12:38:25 UTC
svn commit: r1332612 - in
/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master:
BaseScanner.java ProcessServerShutdown.java RegionServerOperationQueue.java
Author: mbautin
Date: Tue May 1 10:38:24 2012
New Revision: 1332612
URL: http://svn.apache.org/viewvc?rev=1332612&view=rev
Log:
[master] [0.89-fb] Optimise the ProcessServerShutdown -- enable faster meta scans through caching and by reducing lock contention; Avoid starving the delayQueue in the RegionServerOperationQueue
Summary:
1) Add caching to scanMetaRegion. The earlier version used to only get 1 row
at a time.
2) Optimize lock contention, by processing a bunch of in-memory updates at the
end of the loop, instead of grabbing a lock each time.
3) Drain out elements from the delayQueue, whenever they are ready to go, into
the todoQueue to ensure that they do not get starved indefinitely.
Test Plan: run unit tests
Reviewers: kannan, pkhemani
Reviewed By: pkhemani
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D454869
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=1332612&r1=1332611&r2=1332612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java Tue May 1 10:38:24 2012
@@ -178,6 +178,7 @@ abstract class BaseScanner extends Chore
Scan s = new Scan().addFamily(HConstants.CATALOG_FAMILY);
// Make this scan do a row at a time otherwise, data can be stale.
s.setCaching(1);
+ s.setCacheBlocks(true);
scannerId = regionServer.openScanner(metaRegion.getRegionName(), s);
while (true) {
Result values = regionServer.next(scannerId);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=1332612&r1=1332611&r2=1332612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java Tue May 1 10:38:24 2012
@@ -136,79 +136,63 @@ class ProcessServerShutdown extends Regi
/** Finds regions that the dead region server was serving
*/
- protected void scanMetaRegion(HRegionInterface server, long scannerId,
+ protected void scanMetaRegion(HRegionInterface server, Scan scan,
byte [] regionName)
throws IOException {
+ long scannerId = server.openScanner(regionName, scan);
+
+ int rows = scan.getCaching();
+ // The default caching if not set for scans is -1. Handle it
+ if (rows < 1) rows = 1;
+
List<ToDoEntry> toDoList = new ArrayList<ToDoEntry>();
Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
List<byte []> emptyRows = new ArrayList<byte []>();
try {
while (true) {
- Result values = null;
+ Result[] values = null;
try {
- values = server.next(scannerId);
+ values = server.next(scannerId, rows);
} catch (IOException e) {
LOG.error("Shutdown scanning of meta region",
RemoteExceptionHandler.checkIOException(e));
break;
}
- if (values == null || values.size() == 0) {
+ if (values == null || values.length == 0) {
break;
}
- byte [] row = values.getRow();
- // Check server name. If null, skip (We used to consider it was on
- // shutdown server but that would mean that we'd reassign regions that
- // were already out being assigned, ones that were product of a split
- // that happened while the shutdown was being processed).
- String serverAddress = BaseScanner.getServerAddress(values);
- long startCode = BaseScanner.getStartCode(values);
-
- String serverName = null;
- if (serverAddress != null && serverAddress.length() > 0) {
- serverName = HServerInfo.getServerName(serverAddress, startCode);
- }
- if (serverName == null || !deadServer.equals(serverName)) {
- // This isn't the server you're looking for - move along
- continue;
- }
-
- if (LOG.isDebugEnabled() && row != null) {
- LOG.debug("Shutdown scanner for " + serverName + " processing " +
- Bytes.toString(row));
- }
+ for(Result value: values) {
+ if (value.size() == 0) continue;
+ byte [] row = value.getRow();
+ // Check server name. If null, skip (We used to consider it was on
+ // shutdown server but that would mean that we'd reassign regions that
+ // were already out being assigned, ones that were product of a split
+ // that happened while the shutdown was being processed).
+ String serverAddress = BaseScanner.getServerAddress(value);
+ long startCode = BaseScanner.getStartCode(value);
+
+ String serverName = null;
+ if (serverAddress != null && serverAddress.length() > 0) {
+ serverName = HServerInfo.getServerName(serverAddress, startCode);
+ }
+ if (serverName == null || !deadServer.equals(serverName)) {
+ // This isn't the server you're looking for - move along
+ continue;
+ }
- HRegionInfo info = master.getHRegionInfo(row, values);
- if (info == null) {
- emptyRows.add(row);
- continue;
- }
+ if (LOG.isDebugEnabled() && row != null) {
+ LOG.debug("Shutdown scanner for " + serverName + " processing " +
+ Bytes.toString(row));
+ }
- synchronized (master.getRegionManager()) {
- if (info.isMetaTable()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("removing meta region " +
- Bytes.toString(info.getRegionName()) +
- " from online meta regions");
- }
- master.getRegionManager().offlineMetaRegionWithStartKey(info.getStartKey());
+ HRegionInfo info = master.getHRegionInfo(row, value);
+ if (info == null) {
+ emptyRows.add(row);
+ continue;
}
ToDoEntry todo = new ToDoEntry(info);
toDoList.add(todo);
-
- if (master.getRegionManager().isOfflined(info.getRegionNameAsString()) ||
- info.isOffline()) {
- master.getRegionManager().removeRegion(info);
- // Mark region offline
- if (!info.isOffline()) {
- todo.regionOffline = true;
- }
- } else {
- if (!info.isOffline() && !info.isSplit()) {
- // Get region reassigned
- regions.add(info);
- }
- }
}
}
} finally {
@@ -230,7 +214,36 @@ class ProcessServerShutdown extends Regi
Bytes.toString(regionName));
master.deleteEmptyMetaRows(server, regionName, emptyRows);
}
+
+ synchronized (master.getRegionManager()) {
// Update server in root/meta entries
+ for (ToDoEntry e: toDoList) {
+ if (e.info.isMetaTable()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("removing meta region " +
+ Bytes.toString(e.info.getRegionName()) +
+ " from online meta regions");
+ }
+ master.getRegionManager().offlineMetaRegionWithStartKey(e.info.getStartKey());
+ }
+
+ if (master.getRegionManager().isOfflined(e.info.getRegionNameAsString()) ||
+ e.info.isOffline()) {
+ master.getRegionManager().removeRegion(e.info);
+ // Mark region offline
+ if (!e.info.isOffline()) {
+ e.regionOffline = true;
+ }
+ } else {
+ if (!e.info.isOffline() && !e.info.isSplit()) {
+ // Get region reassigned
+ regions.add(e.info);
+ }
+ }
+ }
+ }
+
+ // Let us not do this while we hold the lock on the regionManager.
for (ToDoEntry e: toDoList) {
if (e.regionOffline) {
HRegion.offlineRegionInMETA(server, regionName, e.info);
@@ -260,9 +273,9 @@ class ProcessServerShutdown extends Regi
}
Scan scan = new Scan();
scan.addFamily(HConstants.CATALOG_FAMILY);
- long scannerId = server.openScanner(
- HRegionInfo.ROOT_REGIONINFO.getRegionName(), scan);
- scanMetaRegion(server, scannerId,
+ scan.setCaching(1000);
+ scan.setCacheBlocks(true);
+ scanMetaRegion(server, scan,
HRegionInfo.ROOT_REGIONINFO.getRegionName());
return true;
}
@@ -280,9 +293,9 @@ class ProcessServerShutdown extends Regi
}
Scan scan = new Scan();
scan.addFamily(HConstants.CATALOG_FAMILY);
- long scannerId = server.openScanner(
- m.getRegionName(), scan);
- scanMetaRegion(server, scannerId, m.getRegionName());
+ scan.setCaching(1000);
+ scan.setCacheBlocks(true);
+ scanMetaRegion(server, scan, m.getRegionName());
return true;
}
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java?rev=1332612&r1=1332611&r2=1332612&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java Tue May 1 10:38:24 2012
@@ -121,17 +121,25 @@ public class RegionServerOperationQueue
// Only process the delayed queue if root region is online. If offline,
// the operation to put it online is probably in the toDoQueue. Process
// it first.
- if (toDoQueue.isEmpty()) {
- op = delayedToDoQueue.poll();
- }
- if (op == null) {
+ //
+ // The original logic used to starve the items in the delayed queue.
+ //
+ // Let us add all the ready items in the Delayed queue, behind the items in the
+ // todo queue.
+ while ((op = delayedToDoQueue.poll()) != null) {
try {
- op = toDoQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
+ toDoQueue.put(op);
} catch (InterruptedException e) {
LOG.debug("Interrupted", e);
}
}
+ try {
+ op = toDoQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted", e);
+ }
+
// At this point, if there's still no todo operation, or we're supposed to
// be closed, return.
if (op == null || stop.isStopped()) {