You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2008/02/02 01:44:21 UTC
svn commit: r617723 - in
/hadoop/core/branches/branch-0.16/src/contrib/hbase: ./ conf/
src/java/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/
src/test/org/apache/hadoop/hbase/mapred/
Author: stack
Date: Fri Feb 1 16:44:04 2008
New Revision: 617723
URL: http://svn.apache.org/viewvc?rev=617723&view=rev
Log:
HADOOP-2731 Under load, regions become extremely large and eventually
cause region servers to become unresponsive
Modified:
hadoop/core/branches/branch-0.16/src/contrib/hbase/CHANGES.txt
hadoop/core/branches/branch-0.16/src/contrib/hbase/conf/hbase-default.xml
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreKey.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCompaction.java
hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/CHANGES.txt?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/CHANGES.txt Fri Feb 1 16:44:04 2008
@@ -1,7 +1,6 @@
HBase Change Log
-
-Trunk (unreleased changes)
+Release 0.16.0 - 2008-02-04
INCOMPATIBLE CHANGES
HADOOP-2056 A table with row keys containing colon fails to split regions
@@ -164,6 +163,8 @@
table or table you are enumerating isn't the first table
Delete empty file: src/contrib/hbase/src/java/org/apache/hadoop/hbase/mapred/
TableOutputCollector.java per Nigel Daley
+ HADOOP-2731 Under load, regions become extremely large and eventually cause
+ region servers to become unresponsive
IMPROVEMENTS
HADOOP-2401 Add convenience put method that takes writable
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/conf/hbase-default.xml?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/conf/hbase-default.xml (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/conf/hbase-default.xml Fri Feb 1 16:44:04 2008
@@ -164,7 +164,7 @@
</property>
<property>
<name>hbase.hregion.memcache.block.multiplier</name>
- <value>2</value>
+ <value>1</value>
<description>
Block updates if memcache has hbase.hregion.block.memcache
time hbase.hregion.flush.size bytes. Useful preventing
@@ -179,7 +179,7 @@
<value>268435456</value>
<description>
Maximum HStoreFile size. If any one of a column families' HStoreFiles has
- grown to exceed value + (value / 2), the hosting HRegion is split in two.
+ grown to exceed this value, the hosting HRegion is split in two.
Default: 256M.
</description>
</property>
@@ -200,7 +200,7 @@
</property>
<property>
<name>hbase.regionserver.thread.splitcompactcheckfrequency</name>
- <value>15000</value>
+ <value>20000</value>
<description>How often a region server runs the split/compaction check.
</description>
</property>
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLog.java Fri Feb 1 16:44:04 2008
@@ -262,9 +262,12 @@
break;
}
}
- LOG.debug("Found " + sequenceNumbers.size() + " logs to remove " +
+ if (LOG.isDebugEnabled() && sequenceNumbers.size() > 0) {
+ LOG.debug("Found " + sequenceNumbers.size() +
+ " logs to remove " +
"using oldest outstanding seqnum of " +
oldestOutstandingSeqNum + " from region " + oldestRegion);
+ }
}
if (sequenceNumbers.size() > 0) {
for (Long seq : sequenceNumbers) {
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Fri Feb 1 16:44:04 2008
@@ -218,8 +218,6 @@
volatile boolean compacting = false;
// Gets set in close. If set, cannot compact or flush again.
volatile boolean writesEnabled = true;
- // Gets set in close to prevent new compaction starting
- volatile boolean disableCompactions = false;
}
volatile WriteState writestate = new WriteState();
@@ -319,7 +317,7 @@
1024*1024*64);
this.flushListener = listener;
this.blockingMemcacheSize = this.memcacheFlushSize *
- conf.getInt("hbase.hregion.memcache.block.multiplier", 2);
+ conf.getInt("hbase.hregion.memcache.block.multiplier", 1);
// By default we split region if a file > DEFAULT_MAX_FILE_SIZE.
this.desiredMaxFileSize =
@@ -390,11 +388,11 @@
}
synchronized (splitLock) {
synchronized (writestate) {
- // Can be a compaction running and it can take a long time to
- // complete -- minutes. Meantime, we want flushes to keep happening
- // if we are taking on lots of updates. But we don't want another
- // compaction to start so set disableCompactions flag.
- this.writestate.disableCompactions = true;
+ // Disable compacting and flushing by background threads for this
+ // region.
+ writestate.writesEnabled = false;
+ LOG.debug("compactions and cache flushes disabled for region " +
+ regionName);
while (writestate.compacting || writestate.flushing) {
LOG.debug("waiting for" +
(writestate.compacting ? " compaction" : "") +
@@ -409,11 +407,6 @@
// continue
}
}
- // Disable compacting and flushing by background threads for this
- // region.
- writestate.writesEnabled = false;
- LOG.debug("compactions and cache flushes disabled for region " +
- regionName);
}
lock.writeLock().lock();
LOG.debug("new updates and scanners for region " + regionName +
@@ -671,8 +664,7 @@
(midKey.equals(getStartKey()) && midKey.equals(getEndKey())) ) {
return false;
}
- long triggerSize = this.desiredMaxFileSize + (this.desiredMaxFileSize / 2);
- boolean split = (biggest.getAggregate() >= triggerSize);
+ boolean split = (biggest.getAggregate() >= this.desiredMaxFileSize);
if (split) {
if (!biggest.isSplitable()) {
LOG.warn("Region " + getRegionName().toString() +
@@ -710,10 +702,6 @@
}
}
if (!needsCompaction) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("region " + regionInfo.getRegionName() +
- " does not need compaction");
- }
return false;
}
return compactStores();
@@ -768,15 +756,13 @@
}
try {
synchronized (writestate) {
- if (!writestate.compacting && writestate.writesEnabled &&
- !this.writestate.disableCompactions) {
+ if (!writestate.compacting && writestate.writesEnabled) {
writestate.compacting = true;
} else {
LOG.info("NOT compacting region " +
this.regionInfo.getRegionName().toString() + ": compacting=" +
writestate.compacting + ", writesEnabled=" +
- writestate.writesEnabled + ", writestate.disableCompactions=" +
- this.writestate.disableCompactions);
+ writestate.writesEnabled);
return false;
}
}
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Fri Feb 1 16:44:04 2008
@@ -217,77 +217,49 @@
this.expirationTime = expirationTime;
}
}
-
- // Check to see if regions should be split
- final Splitter splitter;
- // Needed at shutdown. On way out, if can get this lock then we are not in
- // middle of a split or compaction: i.e. splits/compactions cannot be
- // interrupted.
- final Integer splitterLock = new Integer(0);
-
- /** Split regions on request */
- class Splitter extends Thread implements RegionUnavailableListener {
- private final BlockingQueue<QueueEntry> splitQueue =
- new LinkedBlockingQueue<QueueEntry>();
+ // Compactions
+ final CompactSplitThread compactSplitThread;
+ // Needed during shutdown so we send an interrupt after completion of a
+ // compaction, not in the midst.
+ final Integer compactSplitLock = new Integer(0);
+
+ /** Compact region on request and then run split if appropriate
+ */
+ private class CompactSplitThread extends Thread
+ implements RegionUnavailableListener {
private HTable root = null;
private HTable meta = null;
private long startTime;
+ private final long frequency;
+
+ private final BlockingQueue<QueueEntry> compactionQueue =
+ new LinkedBlockingQueue<QueueEntry>();
/** constructor */
- public Splitter() {
+ public CompactSplitThread() {
super();
- }
-
- /** {@inheritDoc} */
- public void closing(final Text regionName) {
- startTime = System.currentTimeMillis();
- lock.writeLock().lock();
- try {
- // Remove region from regions Map and add it to the Map of retiring
- // regions.
- retiringRegions.put(regionName, onlineRegions.remove(regionName));
- if (LOG.isDebugEnabled()) {
- LOG.debug(regionName.toString() + " closing (" +
- "Adding to retiringRegions)");
- }
- } finally {
- lock.writeLock().unlock();
- }
+ this.frequency =
+ conf.getLong("hbase.regionserver.thread.splitcompactcheckfrequency",
+ 20 * 1000);
}
/** {@inheritDoc} */
- public void closed(final Text regionName) {
- lock.writeLock().lock();
- try {
- retiringRegions.remove(regionName);
- if (LOG.isDebugEnabled()) {
- LOG.debug(regionName.toString() + " closed");
- }
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Perform region splits if necessary
- */
@Override
public void run() {
while (!stopRequested.get()) {
QueueEntry e = null;
try {
- e = splitQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
+ e = compactionQueue.poll(this.frequency, TimeUnit.MILLISECONDS);
if (e == null) {
continue;
}
- synchronized (splitterLock) { // Don't interrupt us while we're working
- split(e.getRegion());
- }
+ e.getRegion().compactIfNeeded();
+ split(e.getRegion());
} catch (InterruptedException ex) {
continue;
} catch (IOException ex) {
- LOG.error("Split failed" +
+ LOG.error("Compaction failed" +
(e != null ? (" for region " + e.getRegion().getRegionName()) : ""),
RemoteExceptionHandler.checkIOException(ex));
if (!checkFileSystem()) {
@@ -295,7 +267,7 @@
}
} catch (Exception ex) {
- LOG.error("Split failed" +
+ LOG.error("Compaction failed" +
(e != null ? (" for region " + e.getRegion().getRegionName()) : ""),
ex);
if (!checkFileSystem()) {
@@ -307,18 +279,22 @@
}
/**
- * @param e entry indicating which region needs to be split
+ * @param e QueueEntry for region to be compacted
*/
- public void splitRequested(QueueEntry e) {
- splitQueue.add(e);
+ public void compactionRequested(QueueEntry e) {
+ compactionQueue.add(e);
+ }
+
+ void compactionRequested(final HRegion r) {
+ compactionRequested(new QueueEntry(r, System.currentTimeMillis()));
}
private void split(final HRegion region) throws IOException {
final HRegionInfo oldRegionInfo = region.getRegionInfo();
final HRegion[] newRegions = region.splitRegion(this);
-
if (newRegions == null) {
- return; // Didn't need to be split
+ // Didn't need to be split
+ return;
}
// When a region is split, the META table needs to updated if we're
@@ -374,65 +350,35 @@
// Do not serve the new regions. Let the Master assign them.
}
- }
-
- // Compactions
- final Compactor compactor;
- // Needed during shutdown so we send an interrupt after completion of a
- // compaction, not in the midst.
- final Integer compactionLock = new Integer(0);
-
- /** Compact region on request */
- class Compactor extends Thread {
- private final BlockingQueue<QueueEntry> compactionQueue =
- new LinkedBlockingQueue<QueueEntry>();
-
- /** constructor */
- public Compactor() {
- super();
- }
/** {@inheritDoc} */
- @Override
- public void run() {
- while (!stopRequested.get()) {
- QueueEntry e = null;
- try {
- e = compactionQueue.poll(threadWakeFrequency, TimeUnit.MILLISECONDS);
- if (e == null) {
- continue;
- }
- if (e.getRegion().compactIfNeeded()) {
- splitter.splitRequested(e);
- }
-
- } catch (InterruptedException ex) {
- continue;
- } catch (IOException ex) {
- LOG.error("Compaction failed" +
- (e != null ? (" for region " + e.getRegion().getRegionName()) : ""),
- RemoteExceptionHandler.checkIOException(ex));
- if (!checkFileSystem()) {
- break;
- }
-
- } catch (Exception ex) {
- LOG.error("Compaction failed" +
- (e != null ? (" for region " + e.getRegion().getRegionName()) : ""),
- ex);
- if (!checkFileSystem()) {
- break;
- }
+ public void closing(final Text regionName) {
+ startTime = System.currentTimeMillis();
+ lock.writeLock().lock();
+ try {
+ // Remove region from regions Map and add it to the Map of retiring
+ // regions.
+ retiringRegions.put(regionName, onlineRegions.remove(regionName));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(regionName.toString() + " closing (" +
+ "Adding to retiringRegions)");
}
+ } finally {
+ lock.writeLock().unlock();
}
- LOG.info(getName() + " exiting");
}
- /**
- * @param e QueueEntry for region to be compacted
- */
- public void compactionRequested(QueueEntry e) {
- compactionQueue.add(e);
+ /** {@inheritDoc} */
+ public void closed(final Text regionName) {
+ lock.writeLock().lock();
+ try {
+ retiringRegions.remove(regionName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(regionName.toString() + " closed");
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
}
}
@@ -469,7 +415,7 @@
}
synchronized(cacheFlusherLock) { // Don't interrupt while we're working
if (e.getRegion().flushcache()) {
- compactor.compactionRequested(e);
+ compactSplitThread.compactionRequested(e);
}
e.setExpirationTime(System.currentTimeMillis() +
@@ -650,10 +596,7 @@
this.cacheFlusher = new Flusher();
// Compaction thread
- this.compactor = new Compactor();
-
- // Region split thread
- this.splitter = new Splitter();
+ this.compactSplitThread = new CompactSplitThread();
// Log rolling thread
this.logRoller = new LogRoller();
@@ -846,11 +789,8 @@
synchronized(cacheFlusherLock) {
this.cacheFlusher.interrupt();
}
- synchronized (compactionLock) {
- this.compactor.interrupt();
- }
- synchronized (splitterLock) {
- this.splitter.interrupt();
+ synchronized (compactSplitLock) {
+ this.compactSplitThread.interrupt();
}
synchronized (logRollerLock) {
this.logRoller.interrupt();
@@ -972,9 +912,8 @@
handler);
Threads.setDaemonThreadRunning(this.cacheFlusher, n + ".cacheFlusher",
handler);
- Threads.setDaemonThreadRunning(this.compactor, n + ".compactor",
+ Threads.setDaemonThreadRunning(this.compactSplitThread, n + ".compactor",
handler);
- Threads.setDaemonThreadRunning(this.splitter, n + ".splitter", handler);
Threads.setDaemonThreadRunning(this.workerThread, n + ".worker", handler);
// Leases is not a Thread. Internally it runs a daemon thread. If it gets
// an unhandled exception, it will just exit.
@@ -1038,8 +977,7 @@
void join() {
join(this.workerThread);
join(this.cacheFlusher);
- join(this.compactor);
- join(this.splitter);
+ join(this.compactSplitThread);
join(this.logRoller);
}
@@ -1219,7 +1157,8 @@
),
this.log, this.fs, conf, regionInfo, null, this.cacheFlusher
);
-
+ // Startup a compaction early if one is needed.
+ this.compactSplitThread.compactionRequested(region);
} catch (IOException e) {
LOG.error("error opening region " + regionInfo.getRegionName(), e);
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Fri Feb 1 16:44:04 2008
@@ -1155,15 +1155,22 @@
* @return True if this store needs compaction.
*/
boolean needsCompaction() {
- boolean compactionNeeded = false;
+ return this.storefiles != null &&
+ (this.storefiles.size() >= this.compactionThreshold || hasReferences());
+ }
+
+ /*
+ * @return True if this store has references.
+ */
+ private boolean hasReferences() {
if (this.storefiles != null) {
- compactionNeeded = this.storefiles.size() >= this.compactionThreshold;
- if (LOG.isDebugEnabled()) {
- LOG.debug("compaction for HStore " + storeName +
- (compactionNeeded ? " " : " not ") + "needed.");
+ for (HStoreFile hsf: this.storefiles.values()) {
+ if (hsf.isReference()) {
+ return true;
+ }
}
}
- return compactionNeeded;
+ return false;
}
/**
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreKey.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreKey.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreKey.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreKey.java Fri Feb 1 16:44:04 2008
@@ -68,13 +68,13 @@
/**
* Create an HStoreKey specifying the row and column names
- * The timestamp defaults to Long.MAX_VALUE
+ * The timestamp defaults to LATEST_TIMESTAMP
*
* @param row row key
* @param column column key
*/
public HStoreKey(Text row, Text column) {
- this(row, column, Long.MAX_VALUE);
+ this(row, column, HConstants.LATEST_TIMESTAMP);
}
/**
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java Fri Feb 1 16:44:04 2008
@@ -74,6 +74,9 @@
HRegionInfo parent =
t.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo();
LOG.info("Parent region " + parent.toString());
+ Path parentDir = HRegion.getRegionDir(new Path(d, tableName),
+ parent.getEncodedName());
+ assertTrue(fs.exists(parentDir));
// Now add content.
addContent(new HTableIncommon(t), columnName);
LOG.info("Finished content loading");
@@ -125,72 +128,73 @@
// region we have been dealing with up to this. Its the parent of the
// region split.
Map<Text, byte []> data = getSplitParentInfo(meta, parent);
- parent = Writables.getHRegionInfoOrNull(data.get(HConstants.COL_REGIONINFO));
- LOG.info("Found parent region: " + parent);
- assertTrue(parent.isOffline());
- assertTrue(parent.isSplit());
- HRegionInfo splitA =
- Writables.getHRegionInfoOrNull(data.get(HConstants.COL_SPLITA));
- HRegionInfo splitB =
- Writables.getHRegionInfoOrNull(data.get(HConstants.COL_SPLITB));
- Path parentDir = HRegion.getRegionDir(new Path(d, tableName),
- parent.getEncodedName());
- assertTrue(fs.exists(parentDir));
-
- LOG.info("Split happened. Parent is " + parent.getRegionName());
-
- // Recalibrate will cause us to wait on new regions' deployment
- recalibrate(t, new Text(columnName), retries, waitTime);
-
- if (splitA == null) {
- LOG.info("splitA was already null. Assuming it was previously compacted.");
+ if (data == null) {
+ // We changed stuff so daughters get cleaned up much faster now. Can
+ // run so fast, parent has been deleted by time we get to here.
} else {
- LOG.info("Daughter splitA: " + splitA.getRegionName());
- // Compact a region at a time so we can test case where one region has
- // no references but the other still has some
- compact(cluster, splitA);
-
- // Wait till the parent only has reference to remaining split, one that
- // still has references.
- while (true) {
- data = getSplitParentInfo(meta, parent);
- if (data == null || data.size() == 3) {
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException e) {
- // continue
+ parent = Writables.getHRegionInfoOrNull(data.get(HConstants.COL_REGIONINFO));
+ LOG.info("Found parent region: " + parent);
+ assertTrue(parent.isOffline());
+ assertTrue(parent.isSplit());
+ HRegionInfo splitA =
+ Writables.getHRegionInfoOrNull(data.get(HConstants.COL_SPLITA));
+ HRegionInfo splitB =
+ Writables.getHRegionInfoOrNull(data.get(HConstants.COL_SPLITB));
+ assertTrue(fs.exists(parentDir));
+ LOG.info("Split happened. Parent is " + parent.getRegionName());
+
+ // Recalibrate will cause us to wait on new regions' deployment
+ recalibrate(t, new Text(columnName), retries, waitTime);
+
+ if (splitA == null) {
+ LOG.info("splitA was already null. Assuming it was previously compacted.");
+ } else {
+ LOG.info("Daughter splitA: " + splitA.getRegionName());
+ // Compact a region at a time so we can test case where one region has
+ // no references but the other still has some
+ compact(cluster, splitA);
+
+ // Wait till the parent only has reference to remaining split, one that
+ // still has references.
+ while (true) {
+ data = getSplitParentInfo(meta, parent);
+ if (data == null || data.size() == 3) {
+ try {
+ Thread.sleep(waitTime);
+ } catch (InterruptedException e) {
+ // continue
+ }
+ continue;
}
- continue;
+ break;
}
- break;
+ LOG.info("Parent split info returned " + data.keySet().toString());
}
- LOG.info("Parent split info returned " + data.keySet().toString());
- }
-
- if (splitB == null) {
- LOG.info("splitB was already null. Assuming it was previously compacted.");
- } else {
- LOG.info("Daughter splitB: " + splitA.getRegionName());
- // Call second split.
- compact(cluster, splitB);
- }
-
- // Now wait until parent disappears.
- LOG.info("Waiting on parent " + parent.getRegionName() + " to disappear");
- for (int i = 0; i < retries; i++) {
- if (getSplitParentInfo(meta, parent) == null) {
- break;
- }
-
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException e) {
- // continue
+ if (splitB == null) {
+ LOG.info("splitB was already null. Assuming it was previously compacted.");
+ } else {
+ LOG.info("Daughter splitB: " + splitA.getRegionName());
+
+ // Call second split.
+ compact(cluster, splitB);
+ }
+ // Now wait until parent disappears.
+ LOG.info("Waiting on parent " + parent.getRegionName() + " to disappear");
+ for (int i = 0; i < retries; i++) {
+ if (getSplitParentInfo(meta, parent) == null) {
+ break;
+ }
+
+ try {
+ Thread.sleep(waitTime);
+ } catch (InterruptedException e) {
+ // continue
+ }
}
+ assertNull(getSplitParentInfo(meta, parent));
}
- assertNull(getSplitParentInfo(meta, parent));
-
+
// Assert cleaned up.
for (int i = 0; i < retries; i++) {
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCompaction.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCompaction.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCompaction.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCompaction.java Fri Feb 1 16:44:04 2008
@@ -51,6 +51,7 @@
// Set cache flush size to 1MB
conf.setInt("hbase.hregion.memcache.flush.size", 1024*1024);
+ conf.setInt("hbase.hregion.memcache.block.multiplier", 2);
this.cluster = null;
}
Modified: hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java?rev=617723&r1=617722&r2=617723&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java (original)
+++ hadoop/core/branches/branch-0.16/src/contrib/hbase/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java Fri Feb 1 16:44:04 2008
@@ -48,6 +48,8 @@
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiSearcher;
@@ -83,16 +85,16 @@
/** {@inheritDoc} */
@Override
public void setUp() throws Exception {
+ // Enable DEBUG-level MR logging.
+ Logger.getLogger("org.apache.hadoop.mapred").setLevel(Level.DEBUG);
+
// Make sure the cache gets flushed so we trigger a compaction(s) and
// hence splits.
conf.setInt("hbase.hregion.memcache.flush.size", 1024 * 1024);
-
- // Always compact if there is more than one store file.
- conf.setInt("hbase.hstore.compactionThreshold", 2);
// This size should make it so we always split using the addContent
// below. After adding all data, the first region is 1.3M
- conf.setLong("hbase.hregion.max.filesize", 128 * 1024);
+ conf.setLong("hbase.hregion.max.filesize", 1024 * 1024);
desc = new HTableDescriptor(TABLE_NAME);
desc.addFamily(new HColumnDescriptor(INPUT_COLUMN));