You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2012/12/04 19:14:24 UTC
svn commit: r1417078 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/
Author: jxiang
Date: Tue Dec 4 18:14:23 2012
New Revision: 1417078
URL: http://svn.apache.org/viewvc?rev=1417078&view=rev
Log:
HBASE-6423 Writes should not block reads on blocking updates to memstores
Added:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java (with props)
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java (with props)
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java?rev=1417078&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java Tue Dec 4 18:14:23 2012
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown by a region server if it will block and wait to serve a request.
+ * For example, the client wants to insert something to a region while the
+ * region is compacting.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RegionTooBusyException extends IOException {
+ private static final long serialVersionUID = 1728345723728342L;
+
+ /** default constructor */
+ public RegionTooBusyException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param msg message
+ */
+ public RegionTooBusyException(final String msg) {
+ super(msg);
+ }
+}
Propchange: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java?rev=1417078&r1=1417077&r2=1417078&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java Tue Dec 4 18:14:23 2012
@@ -133,13 +133,6 @@ public class RegionStates {
}
/**
- * @return the server the specified region assigned to; null if not assigned.
- */
- public synchronized ServerName getAssignedServer(final HRegionInfo hri) {
- return regionAssignments.get(hri);
- }
-
- /**
* Wait for the state map to be updated by assignment manager.
*/
public synchronized void waitForUpdate(
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1417078&r1=1417077&r2=1417078&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Dec 4 18:14:23 2012
@@ -59,6 +59,7 @@ import java.util.concurrent.TimeoutExcep
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.protobuf.*;
@@ -84,6 +85,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RegionTooBusyException;
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.client.Append;
@@ -260,6 +262,24 @@ public class HRegion implements HeapSize
private final int rowLockWaitDuration;
static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
+ // The internal wait duration to acquire a lock before read/update
+ // from the region. It is not per row. The purpose of this wait time
+ // is to avoid waiting a long time while the region is busy, so that
+ // we can release the IPC handler soon enough to improve the
+ // availability of the region server. It can be adjusted by
+ // tuning configuration "hbase.busy.wait.duration".
+ final long busyWaitDuration;
+ static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+
+ // If updating multiple rows in one call, wait longer,
+ // i.e. waiting for busyWaitDuration * # of rows. However,
+ // we can limit the max multiplier.
+ final int maxBusyWaitMultiplier;
+
+ // Max busy wait duration. There is no point to wait longer than the RPC
+ // purge timeout, when a RPC call will be terminated by the RPC engine.
+ final long maxBusyWaitDuration;
+
// negative number indicates infinite timeout
static final long DEFAULT_ROW_PROCESSOR_TIMEOUT = 60 * 1000L;
final ExecutorService rowProcessorExecutor = Executors.newCachedThreadPool();
@@ -388,6 +408,9 @@ public class HRegion implements HeapSize
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
+ this.maxBusyWaitDuration = 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+ this.busyWaitDuration = DEFAULT_BUSY_WAIT_DURATION;
+ this.maxBusyWaitMultiplier = 2;
}
/**
@@ -450,6 +473,17 @@ public class HRegion implements HeapSize
this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
+ this.busyWaitDuration = conf.getLong(
+ "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);
+ this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);
+ if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {
+ throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("
+ + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("
+ + maxBusyWaitMultiplier + "). Their product should be positive");
+ }
+ this.maxBusyWaitDuration = conf.getLong("ipc.client.call.purge.timeout",
+ 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+
/*
* timestamp.slop provides a server-side constraint on the timestamp. This
* assumes that you base your TS around currentTimeMillis(). In this case,
@@ -971,6 +1005,7 @@ public class HRegion implements HeapSize
this.closing.set(true);
status.setStatus("Disabling writes for close");
+ // block waiting for the lock for closing
lock.writeLock().lock();
try {
if (this.isClosed()) {
@@ -1170,19 +1205,6 @@ public class HRegion implements HeapSize
return this.lastFlushTime;
}
- /** @return info about the last flushes <time, size> */
- public List<Pair<Long,Long>> getRecentFlushInfo() {
- List<Pair<Long,Long>> ret = null;
- this.lock.readLock().lock();
- try {
- ret = this.recentFlushes;
- this.recentFlushes = new ArrayList<Pair<Long,Long>>();
- } finally {
- this.lock.readLock().unlock();
- }
- return ret;
- }
-
//////////////////////////////////////////////////////////////////////////////
// HRegion maintenance.
//
@@ -1290,6 +1312,7 @@ public class HRegion implements HeapSize
}
Preconditions.checkArgument(cr.getHRegion().equals(this));
MonitoredTask status = null;
+ // block waiting for the lock for compaction
lock.readLock().lock();
try {
status = TaskMonitor.get().createStatus(
@@ -1372,7 +1395,7 @@ public class HRegion implements HeapSize
}
MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
status.setStatus("Acquiring readlock on region");
- lock.readLock().lock();
+ lock(lock.readLock());
try {
if (this.closed.get()) {
LOG.debug("Skipping flush on " + this + " because closed");
@@ -1506,6 +1529,7 @@ public class HRegion implements HeapSize
// end up in both snapshot and memstore (makes it difficult to do atomic
// rows then)
status.setStatus("Obtaining lock to block concurrent updates");
+ // block waiting for the lock for internal flush
this.updatesLock.writeLock().lock();
long flushsize = this.memstoreSize.get();
status.setStatus("Preparing to flush by snapshotting stores");
@@ -2193,7 +2217,7 @@ public class HRegion implements HeapSize
}
}
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock(), numReadyToWrite);
locked = true;
//
@@ -2489,7 +2513,8 @@ public class HRegion implements HeapSize
* this and the synchronize on 'this' inside in internalFlushCache to send
* the notify.
*/
- private void checkResources() {
+ private void checkResources()
+ throws RegionTooBusyException, InterruptedIOException {
// If catalog region, do not impose resource constraints or block updates.
if (this.getRegionInfo().isMetaRegion()) return;
@@ -2507,12 +2532,30 @@ public class HRegion implements HeapSize
" is >= than blocking " +
StringUtils.humanReadableInt(this.blockingMemStoreSize) + " size");
}
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ long timeToWait = startTime + busyWaitDuration - now;
+ if (timeToWait <= 0L) {
+ final long totalTime = now - startTime;
+ this.updatesBlockedMs.add(totalTime);
+ LOG.info("Failed to unblock updates for region " + this + " '"
+ + Thread.currentThread().getName() + "' in " + totalTime
+ + "ms. The region is still busy.");
+ throw new RegionTooBusyException("region is flushing");
+ }
blocked = true;
synchronized(this) {
try {
- wait(threadWakeFrequency);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ wait(Math.min(timeToWait, threadWakeFrequency));
+ } catch (InterruptedException ie) {
+ final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ if (totalTime > 0) {
+ this.updatesBlockedMs.add(totalTime);
+ }
+ LOG.info("Interrupted while waiting to unblock updates for region "
+ + this + " '" + Thread.currentThread().getName() + "'");
+ InterruptedIOException iie = new InterruptedIOException();
+ iie.initCause(ie);
+ throw iie;
}
}
}
@@ -3162,6 +3205,7 @@ public class HRegion implements HeapSize
* @param lockId The lock ID to release.
*/
public void releaseRowLock(final Integer lockId) {
+ if (lockId == null) return; // null lock id, do nothing
HashedBytes rowKey = lockIds.remove(lockId);
if (rowKey == null) {
LOG.warn("Release unknown lockId: " + lockId);
@@ -4372,7 +4416,7 @@ public class HRegion implements HeapSize
acquiredLocks.add(lid);
}
// 3. Region lock
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock(), acquiredLocks.size());
locked = true;
long now = EnvironmentEdgeManager.currentTimeMillis();
@@ -4538,7 +4582,7 @@ public class HRegion implements HeapSize
WriteEntry w = null;
try {
Integer lid = getLock(lockid, row, true);
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock());
// wait for all prior MVCC transactions to finish - while we hold the row lock
// (so that we are guaranteed to see the latest state)
mvcc.completeMemstoreInsert(mvcc.beginMemstoreInsert());
@@ -4704,7 +4748,7 @@ public class HRegion implements HeapSize
WriteEntry w = null;
try {
Integer lid = getLock(lockid, row, true);
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock());
// wait for all prior MVCC transactions to finish - while we hold the row lock
// (so that we are guaranteed to see the latest state)
mvcc.completeMemstoreInsert(mvcc.beginMemstoreInsert());
@@ -4831,8 +4875,8 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
- 40 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
- (7 * Bytes.SIZEOF_LONG) +
+ 40 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+ (8 * Bytes.SIZEOF_LONG) +
Bytes.SIZEOF_BOOLEAN);
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
@@ -5218,13 +5262,16 @@ public class HRegion implements HeapSize
* #closeRegionOperation needs to be called in the try's finally block
* Acquires a read lock and checks if the region is closing or closed.
* @throws NotServingRegionException when the region is closing or closed
+ * @throws RegionTooBusyException if failed to get the lock in time
+ * @throws InterruptedIOException if interrupted while waiting for a lock
*/
- private void startRegionOperation() throws NotServingRegionException {
+ private void startRegionOperation()
+ throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
if (this.closing.get()) {
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closing");
}
- lock.readLock().lock();
+ lock(lock.readLock());
if (this.closed.get()) {
lock.readLock().unlock();
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
@@ -5236,7 +5283,7 @@ public class HRegion implements HeapSize
* Closes the lock. This needs to be called in the finally block corresponding
* to the try block of #startRegionOperation
*/
- private void closeRegionOperation(){
+ private void closeRegionOperation() {
lock.readLock().unlock();
}
@@ -5246,15 +5293,17 @@ public class HRegion implements HeapSize
* #closeBulkRegionOperation needs to be called in the try's finally block
* Acquires a writelock and checks if the region is closing or closed.
* @throws NotServingRegionException when the region is closing or closed
+ * @throws RegionTooBusyException if failed to get the lock in time
+ * @throws InterruptedIOException if interrupted while waiting for a lock
*/
private void startBulkRegionOperation(boolean writeLockNeeded)
- throws NotServingRegionException {
+ throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
if (this.closing.get()) {
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closing");
}
- if (writeLockNeeded) lock.writeLock().lock();
- else lock.readLock().lock();
+ if (writeLockNeeded) lock(lock.writeLock());
+ else lock(lock.readLock());
if (this.closed.get()) {
if (writeLockNeeded) lock.writeLock().unlock();
else lock.readLock().unlock();
@@ -5293,6 +5342,33 @@ public class HRegion implements HeapSize
dataInMemoryWithoutWAL.add(putSize);
}
+ private void lock(final Lock lock)
+ throws RegionTooBusyException, InterruptedIOException {
+ lock(lock, 1);
+ }
+
+ /**
+ * Try to acquire a lock. Throw RegionTooBusyException
+ * if failed to get the lock in time. Throw InterruptedIOException
+ * if interrupted while waiting for the lock.
+ */
+ private void lock(final Lock lock, final int multiplier)
+ throws RegionTooBusyException, InterruptedIOException {
+ try {
+ final long waitTime = Math.min(maxBusyWaitDuration,
+ busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
+ if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
+ throw new RegionTooBusyException(
+ "failed to get a lock in " + waitTime + "ms");
+ }
+ } catch (InterruptedException ie) {
+ LOG.info("Interrupted while waiting for a lock");
+ InterruptedIOException iie = new InterruptedIOException();
+ iie.initCause(ie);
+ throw iie;
+ }
+ }
+
/**
* Calls sync with the given transaction ID if the region's table is not
* deferring it.
@@ -5332,7 +5408,6 @@ public class HRegion implements HeapSize
}
};
-
/**
* Facility for dumping and compacting catalog tables.
* Only does catalog tables since these are only tables we for sure know
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1417078&r1=1417077&r2=1417078&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Tue Dec 4 18:14:23 2012
@@ -18,8 +18,8 @@
*/
package org.apache.hadoop.hbase.regionserver;
-
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -52,8 +52,8 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.MultithreadedTestUtil;
-import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -78,8 +78,8 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
-import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.test.MetricsAssertHelper;
@@ -96,7 +96,6 @@ import org.mockito.Mockito;
import com.google.common.collect.Lists;
-
/**
* Basic stand-alone testing of HRegion.
*
@@ -104,6 +103,7 @@ import com.google.common.collect.Lists;
* HRegions or in the HBaseMaster, so only basic testing is possible.
*/
@Category(MediumTests.class)
+@SuppressWarnings("deprecation")
public class TestHRegion extends HBaseTestCase {
// Do not spin up clusters in here. If you need to spin up a cluster, do it
// over in TestHRegionOnCluster.
@@ -128,8 +128,7 @@ public class TestHRegion extends HBaseTe
protected final byte [] row2 = Bytes.toBytes("rowB");
protected final MetricsAssertHelper metricsAssertHelper =
- CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class);
-
+ CompatibilitySingletonFactory.getInstance(MetricsAssertHelper.class);
/**
* @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
@@ -155,7 +154,6 @@ public class TestHRegion extends HBaseTe
String method = "testCompactionAffectedByScanners";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- Configuration conf = HBaseConfiguration.create();
this.region = initHRegion(tableName, method, conf, family);
Put put = new Put(Bytes.toBytes("r1"));
@@ -207,7 +205,6 @@ public class TestHRegion extends HBaseTe
String method = "testToShowNPEOnRegionScannerReseek";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- Configuration conf = HBaseConfiguration.create();
this.region = initHRegion(tableName, method, conf, family);
Put put = new Put(Bytes.toBytes("r1"));
@@ -240,7 +237,6 @@ public class TestHRegion extends HBaseTe
String method = "testSkipRecoveredEditsReplay";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- Configuration conf = HBaseConfiguration.create();
this.region = initHRegion(tableName, method, conf, family);
try {
Path regiondir = region.getRegionDir();
@@ -293,7 +289,7 @@ public class TestHRegion extends HBaseTe
String method = "testSkipRecoveredEditsReplaySomeIgnored";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+ this.region = initHRegion(tableName, method, conf, family);
try {
Path regiondir = region.getRegionDir();
FileSystem fs = region.getFilesystem();
@@ -350,7 +346,7 @@ public class TestHRegion extends HBaseTe
String method = "testSkipRecoveredEditsReplayAllIgnored";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+ this.region = initHRegion(tableName, method, conf, family);
try {
Path regiondir = region.getRegionDir();
FileSystem fs = region.getFilesystem();
@@ -482,7 +478,7 @@ public class TestHRegion extends HBaseTe
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
- this.region = initHRegion(TABLE, getName(), FAMILIES);
+ this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
try {
String value = "this is the value";
String value2 = "this is some other value";
@@ -603,7 +599,7 @@ public class TestHRegion extends HBaseTe
public void testFamilyWithAndWithoutColon() throws Exception {
byte [] b = Bytes.toBytes(getName());
byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
- this.region = initHRegion(b, getName(), cf);
+ this.region = initHRegion(b, getName(), conf, cf);
try {
Put p = new Put(b);
byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
@@ -627,7 +623,7 @@ public class TestHRegion extends HBaseTe
byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
byte[] qual = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("val");
- this.region = initHRegion(b, getName(), cf);
+ this.region = initHRegion(b, getName(), conf, cf);
MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
try {
long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
@@ -664,7 +660,7 @@ public class TestHRegion extends HBaseTe
Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
MultithreadedTestUtil.TestContext ctx =
- new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+ new MultithreadedTestUtil.TestContext(conf);
final AtomicReference<OperationStatus[]> retFromThread =
new AtomicReference<OperationStatus[]>();
TestThread putter = new TestThread(ctx) {
@@ -731,9 +727,7 @@ public class TestHRegion extends HBaseTe
byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
byte[] qual = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("val");
-
- HBaseConfiguration conf = new HBaseConfiguration();
-
+ Configuration conf = HBaseConfiguration.create(this.conf);
// add data with a timestamp that is too recent for range. Ensure assert
conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -780,7 +774,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting empty data in key
Put put = new Put(row1);
@@ -855,7 +849,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in key
Put put = new Put(row1);
@@ -889,7 +883,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in key
Put put = new Put(row1);
@@ -927,7 +921,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in the key to check
Put put = new Put(row1);
@@ -966,7 +960,7 @@ public class TestHRegion extends HBaseTe
}
public void testCheckAndPut_wrongRowInPut() throws IOException {
- this.region = initHRegion(tableName, this.getName(), COLUMNS);
+ this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
try {
Put put = new Put(row2);
put.add(fam1, qual1, value1);
@@ -1001,7 +995,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Put content
Put put = new Put(row1);
@@ -1076,7 +1070,7 @@ public class TestHRegion extends HBaseTe
put.add(fam1, qual, 2, value);
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
region.put(put);
@@ -1106,7 +1100,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1, fam2, fam3);
+ this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
try {
List<KeyValue> kvs = new ArrayList<KeyValue>();
kvs.add(new KeyValue(row1, fam4, null, null));
@@ -1144,7 +1138,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("info");
byte [][] families = {fam};
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
@@ -1212,7 +1206,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("info");
byte [][] families = {fam};
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
byte [] row = Bytes.toBytes("table_name");
// column names
@@ -1255,7 +1249,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("info");
byte [][] families = {fam};
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
byte [] row = Bytes.toBytes("row1");
// column names
@@ -1308,7 +1302,7 @@ public class TestHRegion extends HBaseTe
byte[] fam = Bytes.toBytes("info");
byte[][] families = { fam };
String method = this.getName();
- HBaseConfiguration conf = new HBaseConfiguration();
+ Configuration conf = HBaseConfiguration.create(this.conf);
// add data with a timestamp that is too recent for range. Ensure assert
conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -1339,7 +1333,7 @@ public class TestHRegion extends HBaseTe
byte [] tableName = Bytes.toBytes("test_table");
byte [] fam1 = Bytes.toBytes("columnA");
byte [] fam2 = Bytes.toBytes("columnB");
- this.region = initHRegion(tableName, getName(), fam1, fam2);
+ this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
try {
byte [] rowA = Bytes.toBytes("rowA");
byte [] rowB = Bytes.toBytes("rowB");
@@ -1392,7 +1386,7 @@ public class TestHRegion extends HBaseTe
public void doTestDelete_AndPostInsert(Delete delete)
throws IOException, InterruptedException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
Put put = new Put(row);
@@ -1445,7 +1439,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Building checkerList
List<KeyValue> kvs = new ArrayList<KeyValue>();
@@ -1485,7 +1479,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
Get get = new Get(row1);
get.addColumn(fam2, col1);
@@ -1516,7 +1510,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Add to memstore
Put put = new Put(row1);
@@ -1566,7 +1560,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("fam");
String method = this.getName();
- this.region = initHRegion(tableName, method, fam);
+ this.region = initHRegion(tableName, method, conf, fam);
try {
Get get = new Get(row);
get.addFamily(fam);
@@ -1586,7 +1580,8 @@ public class TestHRegion extends HBaseTe
public void stestGet_Root() throws IOException {
//Setting up region
String method = this.getName();
- this.region = initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
+ this.region = initHRegion(HConstants.ROOT_TABLE_NAME,
+ method, conf, HConstants.CATALOG_FAMILY);
try {
//Add to memstore
Put put = new Put(HConstants.EMPTY_START_ROW);
@@ -1817,7 +1812,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
Scan scan = new Scan();
scan.addFamily(fam1);
@@ -1842,7 +1837,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
Scan scan = new Scan();
scan.addFamily(fam2);
@@ -1871,7 +1866,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
@@ -1919,7 +1914,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
try {
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
} catch (IOException e) {
e.printStackTrace();
fail("Got IOException during initHRegion, " + e.getMessage());
@@ -1955,7 +1950,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2022,7 +2017,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2082,7 +2077,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2147,7 +2142,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2229,7 +2224,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2290,7 +2285,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in Region
Put put = null;
@@ -2341,7 +2336,7 @@ public class TestHRegion extends HBaseTe
public void testScanner_StopRow1542() throws IOException {
byte [] tableName = Bytes.toBytes("test_table");
byte [] family = Bytes.toBytes("testFamily");
- this.region = initHRegion(tableName, getName(), family);
+ this.region = initHRegion(tableName, getName(), conf, family);
try {
byte [] row1 = Bytes.toBytes("row111");
byte [] row2 = Bytes.toBytes("row222");
@@ -2432,7 +2427,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in Region
KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2647,7 +2642,7 @@ public class TestHRegion extends HBaseTe
int compactInterval = 10 * flushAndScanInterval;
String method = "testFlushCacheWhileScanning";
- this.region = initHRegion(tableName,method, family);
+ this.region = initHRegion(tableName,method, conf, family);
try {
FlushThread flushThread = new FlushThread();
flushThread.start();
@@ -2778,7 +2773,7 @@ public class TestHRegion extends HBaseTe
}
String method = "testWritesWhileScanning";
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
PutThread putThread = new PutThread(numRows, families, qualifiers);
putThread.start();
@@ -2900,6 +2895,8 @@ public class TestHRegion extends HBaseTe
}
numPutsFinished++;
}
+ } catch (InterruptedIOException e) {
+ // This is fine. It means we are done, or didn't get the lock on time
} catch (IOException e) {
LOG.error("error while putting records", e);
error = e;
@@ -2936,8 +2933,9 @@ public class TestHRegion extends HBaseTe
qualifiers[i] = Bytes.toBytes("qual" + i);
}
+ Configuration conf = HBaseConfiguration.create(this.conf);
+
String method = "testWritesWhileGetting";
- Configuration conf = HBaseConfiguration.create();
// This test flushes constantly and can cause many files to be created, possibly
// extending over the ulimit. Make sure compactions are aggressive in reducing
// the number of HFiles created.
@@ -2946,7 +2944,7 @@ public class TestHRegion extends HBaseTe
this.region = initHRegion(tableName, method, conf, families);
PutThread putThread = null;
MultithreadedTestUtil.TestContext ctx =
- new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+ new MultithreadedTestUtil.TestContext(conf);
try {
putThread = new PutThread(numRows, families, qualifiers);
putThread.start();
@@ -3032,7 +3030,7 @@ public class TestHRegion extends HBaseTe
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
- HBaseConfiguration.create(), family);
+ conf, family);
try {
byte[] rowNotServed = Bytes.toBytes("a");
Get g = new Get(rowNotServed);
@@ -3057,7 +3055,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = "testIndexesScanWithOneDeletedRow";
- this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+ this.region = initHRegion(tableName, method, conf, family);
try {
Put put = new Put(Bytes.toBytes(1L));
put.add(family, qual1, 1L, Bytes.toBytes(1L));
@@ -3388,7 +3386,6 @@ public class TestHRegion extends HBaseTe
*/
@Test
public void testParallelIncrementWithMemStoreFlush() throws Exception {
- Configuration conf = HBaseConfiguration.create();
String method = "testParallelIncrementWithMemStoreFlush";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Incrementer.family;
@@ -3480,7 +3477,6 @@ public class TestHRegion extends HBaseTe
*/
@Test
public void testParallelAppendWithMemStoreFlush() throws Exception {
- Configuration conf = HBaseConfiguration.create();
String method = "testParallelAppendWithMemStoreFlush";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Appender.family;
@@ -3542,7 +3538,6 @@ public class TestHRegion extends HBaseTe
* @throws Exception
*/
public void testPutWithMemStoreFlush() throws Exception {
- Configuration conf = HBaseConfiguration.create();
String method = "testPutWithMemStoreFlush";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");;
@@ -3679,7 +3674,8 @@ public class TestHRegion extends HBaseTe
}
private Configuration initSplit() {
- Configuration conf = HBaseConfiguration.create();
+ Configuration conf = HBaseConfiguration.create(this.conf);
+
// Always compact if there is more than one store file.
conf.setInt("hbase.hstore.compactionThreshold", 2);
@@ -3701,19 +3697,6 @@ public class TestHRegion extends HBaseTe
/**
* @param tableName
* @param callingMethod
- * @param families
- * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
- * @throws IOException
- */
- private static HRegion initHRegion (byte [] tableName, String callingMethod,
- byte[] ... families)
- throws IOException {
- return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
- }
-
- /**
- * @param tableName
- * @param callingMethod
* @param conf
* @param families
* @throws IOException
@@ -3771,7 +3754,5 @@ public class TestHRegion extends HBaseTe
assertEquals("Value mismatch while checking: " + ctx,
"value-version-" + ts, Bytes.toString(kv.getValue()));
}
-
-
}
Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java?rev=1417078&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java Tue Dec 4 18:14:23 2012
@@ -0,0 +1,90 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.RegionTooBusyException;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * TestHRegion with hbase.busy.wait.duration set to 1000 (1 second).
+ * We can't use parameterized test since TestHRegion is old fashion.
+ */
+@Category(MediumTests.class)
+@SuppressWarnings("deprecation")
+public class TestHRegionBusyWait extends TestHRegion {
+ public TestHRegionBusyWait() {
+ conf.set("hbase.busy.wait.duration", "1000");
+ }
+
+ /**
+ * Test RegionTooBusyException thrown when region is busy
+ */
+ @Test (timeout=2000)
+ public void testRegionTooBusy() throws IOException {
+ String method = "testRegionTooBusy";
+ byte[] tableName = Bytes.toBytes(method);
+ byte[] family = Bytes.toBytes("family");
+ region = initHRegion(tableName, method, conf, family);
+ final AtomicBoolean stopped = new AtomicBoolean(true);
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ region.lock.writeLock().lock();
+ stopped.set(false);
+ while (!stopped.get()) {
+ Thread.sleep(100);
+ }
+ } catch (InterruptedException ie) {
+ } finally {
+ region.lock.writeLock().unlock();
+ }
+ }
+ });
+ t.start();
+ Get get = new Get(row);
+ try {
+ while (stopped.get()) {
+ Thread.sleep(100);
+ }
+ region.get(get, null);
+ fail("Should throw RegionTooBusyException");
+ } catch (InterruptedException ie) {
+ fail("test interrupted");
+ } catch (RegionTooBusyException e) {
+ // Good, expected
+ } finally {
+ stopped.set(true);
+ try {
+ t.join();
+ } catch (Throwable e) {
+ }
+
+ HRegion.closeHRegion(region);
+ region = null;
+ }
+ }
+}
Propchange: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
------------------------------------------------------------------------------
svn:eol-style = native