You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2008/11/11 22:56:57 UTC
svn commit: r713189 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/mapred/
src/java/org/apache/hadoop/hbase/master/
src/java/org/apache/hadoop/hbase/regionserver/
Author: stack
Date: Tue Nov 11 13:56:55 2008
New Revision: 713189
URL: http://svn.apache.org/viewvc?rev=713189&view=rev
Log:
HBASE-993 Turn of logging of every catalog table row entry on every scan
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Nov 11 13:56:55 2008
@@ -111,6 +111,7 @@
HBASE-983 Declare Perl namespace in Hbase.thrift
HBASE-987 We need a Hbase Partitioner for TableMapReduceUtil.initTableReduceJob
MR Jobs (Billy Pearson via Stack)
+ HBASE-993 Turn of logging of every catalog table row entry on every scan
NEW FEATURES
HBASE-875 Use MurmurHash instead of JenkinsHash [in bloomfilters]
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java Tue Nov 11 13:56:55 2008
@@ -102,4 +102,4 @@
}
}
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/BaseScanner.java Tue Nov 11 13:56:55 2008
@@ -154,11 +154,11 @@
Map<HRegionInfo, RowResult> splitParents =
new HashMap<HRegionInfo, RowResult>();
List<byte []> emptyRows = new ArrayList<byte []>();
+ int rows = 0;
try {
regionServer = master.connection.getHRegionConnection(region.getServer());
scannerId = regionServer.openScanner(region.getRegionName(),
COLUMN_FAMILY_ARRAY, EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP, null);
- int numberOfRegionsFound = 0;
while (true) {
RowResult values = regionServer.next(scannerId);
if (values == null || values.size() == 0) {
@@ -171,20 +171,16 @@
}
String serverName = Writables.cellToString(values.get(COL_SERVER));
long startCode = Writables.cellToLong(values.get(COL_STARTCODE));
- if (LOG.isDebugEnabled()) {
- LOG.debug(Thread.currentThread().getName() + " " + info.toString() +
- "}, SERVER => '" + serverName + "', STARTCODE => " + startCode);
- }
// Note Region has been assigned.
checkAssigned(info, serverName, startCode);
if (isSplitParent(info)) {
splitParents.put(info, values);
}
- numberOfRegionsFound += 1;
+ rows += 1;
}
if (rootRegion) {
- regionManager.setNumMetaRegions(numberOfRegionsFound);
+ regionManager.setNumMetaRegions(rows);
}
} catch (IOException e) {
if (e instanceof RemoteException) {
@@ -226,8 +222,8 @@
cleanupSplits(region.getRegionName(), regionServer, hri, e.getValue());
}
}
- LOG.info(Thread.currentThread().getName() + " scan of meta region " +
- region.toString() + " complete");
+ LOG.info(Thread.currentThread().getName() + " scan of " + rows +
+ " row(s) of meta region " + region.toString() + " complete");
}
/*
@@ -427,4 +423,4 @@
}
}
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaRegion.java Tue Nov 11 13:56:55 2008
@@ -97,4 +97,4 @@
}
return result;
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/MetaScanner.java Tue Nov 11 13:56:55 2008
@@ -57,15 +57,14 @@
// Don't retry if we get an error while scanning. Errors are most often
// caused by the server going away. Wait until next rescan interval when
- // things should be back to normal
+ // things should be back to normal.
private boolean scanOneMetaRegion(MetaRegion region) {
- boolean scanSuccessful = false;
while (!master.closed.get() && !regionManager.isInitialRootScanComplete() &&
regionManager.getRootRegionLocation() == null) {
master.sleeper.sleep();
}
if (master.closed.get()) {
- return scanSuccessful;
+ return false;
}
try {
@@ -74,7 +73,6 @@
scanRegion(region);
regionManager.putMetaRegionOnline(region);
}
- scanSuccessful = true;
} catch (IOException e) {
e = RemoteExceptionHandler.checkIOException(e);
LOG.warn("Scan one META region: " + region.toString(), e);
@@ -85,7 +83,7 @@
if (!regionManager.isMetaRegionOnline(region.getStartKey())) {
LOG.debug("Scanned region is no longer in map of online " +
"regions or its value has changed");
- return scanSuccessful;
+ return false;
}
// Make sure the file system is still available
master.checkFileSystem();
@@ -94,7 +92,7 @@
// at least log it rather than go out silently.
LOG.error("Unexpected exception", e);
}
- return scanSuccessful;
+ return true;
}
@Override
@@ -125,22 +123,27 @@
@Override
protected void maintenanceScan() {
List<MetaRegion> regions = regionManager.getListOfOnlineMetaRegions();
+ int regionCount = 0;
for (MetaRegion r: regions) {
scanOneMetaRegion(r);
+ regionCount++;
}
+ LOG.info("All " + regionCount + " .META. region(s) scanned");
metaRegionsScanned();
}
- /**
+ /*
* Called by the meta scanner when it has completed scanning all meta
* regions. This wakes up any threads that were waiting for this to happen.
+ * @param totalRows Total rows scanned.
+ * @param regionCount Count of regions in .META. table.
+ * @return False if number of meta regions matches count of online regions.
*/
private synchronized boolean metaRegionsScanned() {
if (!regionManager.isInitialRootScanComplete() ||
regionManager.numMetaRegions() != regionManager.numOnlineMetaRegions()) {
return false;
}
- LOG.info("all meta regions scanned");
notifyAll();
return true;
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RootScanner.java Tue Nov 11 13:56:55 2008
@@ -36,14 +36,17 @@
super(master, regionManager, true, master.metaRescanInterval, master.closed);
}
- // Don't retry if we get an error while scanning. Errors are most often
- // caused by the server going away. Wait until next rescan interval when
- // things should be back to normal
+ /*
+ * Don't retry if we get an error while scanning. Errors are most often
+ *
+ * caused by the server going away. Wait until next rescan interval when
+ * things should be back to normal.
+ * @return True if successfully scanned.
+ */
private boolean scanRoot() {
- boolean scanSuccessful = false;
master.waitForRootRegionLocation();
if (master.closed.get()) {
- return scanSuccessful;
+ return false;
}
try {
@@ -54,7 +57,6 @@
HRegionInfo.ROOT_REGIONINFO.getRegionName()));
}
}
- scanSuccessful = true;
} catch (IOException e) {
e = RemoteExceptionHandler.checkIOException(e);
LOG.warn("Scan ROOT region", e);
@@ -65,12 +67,12 @@
// at least log it rather than go out silently.
LOG.error("Unexpected exception", e);
}
- return scanSuccessful;
+ return true;
}
@Override
protected boolean initialScan() {
- initialScanComplete = scanRoot();
+ this.initialScanComplete = scanRoot();
return initialScanComplete;
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/ServerManager.java Tue Nov 11 13:56:55 2008
@@ -87,7 +87,7 @@
master.getConfiguration().getInt("hbase.master.lease.thread.wakefrequency",
15 * 1000));
this.loggingPeriodForAverageLoad = master.getConfiguration().
- getLong("hbase.master.avgload.logging.period", 15000);
+ getLong("hbase.master.avgload.logging.period", 60000);
}
/**
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=713189&r1=713188&r2=713189&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Nov 11 13:56:55 2008
@@ -369,7 +369,7 @@
// Disable compacting and flushing by background threads for this
// region.
writestate.writesEnabled = false;
- LOG.debug("Compactions and cache flushes disabled for region " + this);
+ LOG.debug("Closing " + this + ": compactions & flushes disabled ");
while (writestate.compacting || writestate.flushing) {
LOG.debug("waiting for" +
(writestate.compacting ? " compaction" : "") +
@@ -384,7 +384,6 @@
}
}
newScannerLock.writeLock().lock();
- LOG.debug("Scanners disabled for region " + this);
try {
// Wait for active scanners to finish. The write lock we hold will
// prevent new scanners from being created.
@@ -399,9 +398,9 @@
}
}
}
- LOG.debug("No more active scanners for region " + this);
splitsAndClosesLock.writeLock().lock();
- LOG.debug("Updates disabled for region " + this);
+ LOG.debug("Updates disabled for region, no outstanding scanners on " +
+ this);
try {
// Write lock means no more row locks can be given out. Wait on
// outstanding row locks to come in before we close so we do not drop
@@ -420,7 +419,7 @@
}
this.closed.set(true);
- LOG.info("closed " + this);
+ LOG.info("Closed " + this);
return result;
} finally {
splitsAndClosesLock.writeLock().unlock();