You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2012/12/04 19:16:23 UTC
svn commit: r1417079 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/
Author: jxiang
Date: Tue Dec 4 18:16:21 2012
New Revision: 1417079
URL: http://svn.apache.org/viewvc?rev=1417079&view=rev
Log:
HBASE-6423 Writes should not block reads on blocking updates to memstores
Added:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java (with props)
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java (with props)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Added: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java?rev=1417079&view=auto
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java (added)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java Tue Dec 4 18:16:21 2012
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Thrown by a region server if it will block and wait to serve a request.
+ * For example, the client wants to insert something to a region while the
+ * region is compacting.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class RegionTooBusyException extends IOException {
+ private static final long serialVersionUID = 1728345723728342L;
+
+ /** default constructor */
+ public RegionTooBusyException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param msg message
+ */
+ public RegionTooBusyException(final String msg) {
+ super(msg);
+ }
+}
Propchange: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/RegionTooBusyException.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1417079&r1=1417078&r2=1417079&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Dec 4 18:16:21 2012
@@ -55,6 +55,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
@@ -76,6 +77,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.RegionTooBusyException;
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.backup.HFileArchiver;
import org.apache.hadoop.hbase.client.Append;
@@ -235,6 +237,25 @@ public class HRegion implements HeapSize
final Configuration conf;
final int rowLockWaitDuration;
static final int DEFAULT_ROWLOCK_WAIT_DURATION = 30000;
+
+ // The internal wait duration to acquire a lock before read/update
+ // from the region. It is not per row. The purpose of this wait time
+ // is to avoid waiting a long time while the region is busy, so that
+ // we can release the IPC handler soon enough to improve the
+ // availability of the region server. It can be adjusted by
+ // tuning configuration "hbase.busy.wait.duration".
+ final long busyWaitDuration;
+ static final long DEFAULT_BUSY_WAIT_DURATION = HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+
+ // If updating multiple rows in one call, wait longer,
+ // i.e. waiting for busyWaitDuration * # of rows. However,
+ // we can limit the max multiplier.
+ final int maxBusyWaitMultiplier;
+
+ // Max busy wait duration. There is no point to wait longer than the RPC
+ // purge timeout, when a RPC call will be terminated by the RPC engine.
+ final long maxBusyWaitDuration;
+
final HRegionInfo regionInfo;
final Path regiondir;
KeyValue.KVComparator comparator;
@@ -354,6 +375,10 @@ public class HRegion implements HeapSize
this.coprocessorHost = null;
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
this.opMetrics = new OperationMetrics();
+
+ this.maxBusyWaitDuration = 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT;
+ this.busyWaitDuration = DEFAULT_BUSY_WAIT_DURATION;
+ this.maxBusyWaitMultiplier = 2;
}
/**
@@ -400,6 +425,17 @@ public class HRegion implements HeapSize
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
this.opMetrics = new OperationMetrics(conf, this.regionInfo);
+ this.busyWaitDuration = conf.getLong(
+ "hbase.busy.wait.duration", DEFAULT_BUSY_WAIT_DURATION);
+ this.maxBusyWaitMultiplier = conf.getInt("hbase.busy.wait.multiplier.max", 2);
+ if (busyWaitDuration * maxBusyWaitMultiplier <= 0L) {
+ throw new IllegalArgumentException("Invalid hbase.busy.wait.duration ("
+ + busyWaitDuration + ") or hbase.busy.wait.multiplier.max ("
+ + maxBusyWaitMultiplier + "). Their product should be positive");
+ }
+ this.maxBusyWaitDuration = conf.getLong("ipc.client.call.purge.timeout",
+ 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
+
/*
* timestamp.slop provides a server-side constraint on the timestamp. This
* assumes that you base your TS around currentTimeMillis(). In this case,
@@ -883,6 +919,7 @@ public class HRegion implements HeapSize
this.closing.set(true);
status.setStatus("Disabling writes for close");
+ // block waiting for the lock for closing
lock.writeLock().lock();
try {
if (this.isClosed()) {
@@ -1192,6 +1229,7 @@ public class HRegion implements HeapSize
return false;
}
Preconditions.checkArgument(cr.getHRegion().equals(this));
+ // block waiting for the lock for compaction
lock.readLock().lock();
MonitoredTask status = TaskMonitor.get().createStatus(
"Compacting " + cr.getStore() + " in " + this);
@@ -1271,7 +1309,7 @@ public class HRegion implements HeapSize
}
MonitoredTask status = TaskMonitor.get().createStatus("Flushing " + this);
status.setStatus("Acquiring readlock on region");
- lock.readLock().lock();
+ lock(lock.readLock());
try {
if (this.closed.get()) {
LOG.debug("Skipping flush on " + this + " because closed");
@@ -1406,6 +1444,7 @@ public class HRegion implements HeapSize
// end up in both snapshot and memstore (makes it difficult to do atomic
// rows then)
status.setStatus("Obtaining lock to block concurrent updates");
+ // block waiting for the lock for internal flush
this.updatesLock.writeLock().lock();
long flushsize = this.memstoreSize.get();
status.setStatus("Preparing to flush by snapshotting stores");
@@ -1784,7 +1823,7 @@ public class HRegion implements HeapSize
byte [] byteNow = Bytes.toBytes(now);
boolean flush = false;
- updatesLock.readLock().lock();
+ lock(updatesLock.readLock());
try {
prepareDeleteTimestamps(delete.getFamilyMap(), byteNow);
@@ -2163,7 +2202,7 @@ public class HRegion implements HeapSize
}
}
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock(), numReadyToWrite);
locked = true;
//
@@ -2457,7 +2496,8 @@ public class HRegion implements HeapSize
* this and the synchronize on 'this' inside in internalFlushCache to send
* the notify.
*/
- private void checkResources() {
+ private void checkResources()
+ throws RegionTooBusyException, InterruptedIOException {
// If catalog region, do not impose resource constraints or block updates.
if (this.getRegionInfo().isMetaRegion()) return;
@@ -2475,12 +2515,30 @@ public class HRegion implements HeapSize
" is >= than blocking " +
StringUtils.humanReadableInt(this.blockingMemStoreSize) + " size");
}
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ long timeToWait = startTime + busyWaitDuration - now;
+ if (timeToWait <= 0L) {
+ final long totalTime = now - startTime;
+ this.updatesBlockedMs.add(totalTime);
+ LOG.info("Failed to unblock updates for region " + this + " '"
+ + Thread.currentThread().getName() + "' in " + totalTime
+ + "ms. The region is still busy.");
+ throw new RegionTooBusyException("region is flushing");
+ }
blocked = true;
synchronized(this) {
try {
- wait(threadWakeFrequency);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ wait(Math.min(timeToWait, threadWakeFrequency));
+ } catch (InterruptedException ie) {
+ final long totalTime = EnvironmentEdgeManager.currentTimeMillis() - startTime;
+ if (totalTime > 0) {
+ this.updatesBlockedMs.add(totalTime);
+ }
+ LOG.info("Interrupted while waiting to unblock updates for region "
+ + this + " '" + Thread.currentThread().getName() + "'");
+ InterruptedIOException iie = new InterruptedIOException();
+ iie.initCause(ie);
+ throw iie;
}
}
}
@@ -2547,7 +2605,7 @@ public class HRegion implements HeapSize
byte[] byteNow = Bytes.toBytes(now);
boolean flush = false;
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock());
try {
checkFamilies(familyMap.keySet());
checkTimestamps(familyMap, now);
@@ -3172,6 +3230,7 @@ public class HRegion implements HeapSize
* @param lockId The lock ID to release.
*/
public void releaseRowLock(final Integer lockId) {
+ if (lockId == null) return; // null lock id, do nothing
HashedBytes rowKey = lockIds.remove(lockId);
if (rowKey == null) {
LOG.warn("Release unknown lockId: " + lockId);
@@ -4303,7 +4362,7 @@ public class HRegion implements HeapSize
}
// 3. acquire the region lock
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock(), acquiredLocks.size());
locked = true;
// 4. Get a mvcc write number
@@ -4456,7 +4515,7 @@ public class HRegion implements HeapSize
this.writeRequestsCount.increment();
try {
Integer lid = getLock(lockid, row, true);
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock());
try {
long now = EnvironmentEdgeManager.currentTimeMillis();
// Process each family
@@ -4607,7 +4666,7 @@ public class HRegion implements HeapSize
this.writeRequestsCount.increment();
try {
Integer lid = getLock(lockid, row, true);
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock());
try {
long now = EnvironmentEdgeManager.currentTimeMillis();
// Process each family
@@ -4725,7 +4784,7 @@ public class HRegion implements HeapSize
this.writeRequestsCount.increment();
try {
Integer lid = obtainRowLock(row);
- this.updatesLock.readLock().lock();
+ lock(this.updatesLock.readLock());
try {
Store store = stores.get(family);
@@ -5119,13 +5178,16 @@ public class HRegion implements HeapSize
* #closeRegionOperation needs to be called in the try's finally block
* Acquires a read lock and checks if the region is closing or closed.
* @throws NotServingRegionException when the region is closing or closed
+ * @throws RegionTooBusyException if failed to get the lock in time
+ * @throws InterruptedIOException if interrupted while waiting for a lock
*/
- private void startRegionOperation() throws NotServingRegionException {
+ private void startRegionOperation()
+ throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
if (this.closing.get()) {
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closing");
}
- lock.readLock().lock();
+ lock(lock.readLock());
if (this.closed.get()) {
lock.readLock().unlock();
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
@@ -5147,15 +5209,17 @@ public class HRegion implements HeapSize
* #closeBulkRegionOperation needs to be called in the try's finally block
* Acquires a writelock and checks if the region is closing or closed.
* @throws NotServingRegionException when the region is closing or closed
+ * @throws RegionTooBusyException if failed to get the lock in time
+ * @throws InterruptedIOException if interrupted while waiting for a lock
*/
private void startBulkRegionOperation(boolean writeLockNeeded)
- throws NotServingRegionException {
+ throws NotServingRegionException, RegionTooBusyException, InterruptedIOException {
if (this.closing.get()) {
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closing");
}
- if (writeLockNeeded) lock.writeLock().lock();
- else lock.readLock().lock();
+ if (writeLockNeeded) lock(lock.writeLock());
+ else lock(lock.readLock());
if (this.closed.get()) {
if (writeLockNeeded) lock.writeLock().unlock();
else lock.readLock().unlock();
@@ -5168,7 +5232,7 @@ public class HRegion implements HeapSize
* Closes the lock. This needs to be called in the finally block corresponding
* to the try block of #startRegionOperation
*/
- private void closeBulkRegionOperation(){
+ private void closeBulkRegionOperation() {
if (lock.writeLock().isHeldByCurrentThread()) lock.writeLock().unlock();
else lock.readLock().unlock();
}
@@ -5193,6 +5257,33 @@ public class HRegion implements HeapSize
dataInMemoryWithoutWAL.addAndGet(putSize);
}
+ private void lock(final Lock lock)
+ throws RegionTooBusyException, InterruptedIOException {
+ lock(lock, 1);
+ }
+
+ /**
+ * Try to acquire a lock. Throw RegionTooBusyException
+ * if failed to get the lock in time. Throw InterruptedIOException
+ * if interrupted while waiting for the lock.
+ */
+ private void lock(final Lock lock, final int multiplier)
+ throws RegionTooBusyException, InterruptedIOException {
+ try {
+ final long waitTime = Math.min(maxBusyWaitDuration,
+ busyWaitDuration * Math.min(multiplier, maxBusyWaitMultiplier));
+ if (!lock.tryLock(waitTime, TimeUnit.MILLISECONDS)) {
+ throw new RegionTooBusyException(
+ "failed to get a lock in " + waitTime + "ms");
+ }
+ } catch (InterruptedException ie) {
+ LOG.info("Interrupted while waiting for a lock");
+ InterruptedIOException iie = new InterruptedIOException();
+ iie.initCause(ie);
+ throw iie;
+ }
+ }
+
/**
* Calls sync with the given transaction ID if the region's table is not
* deferring it.
@@ -5232,7 +5323,6 @@ public class HRegion implements HeapSize
}
};
-
/**
* Facility for dumping and compacting catalog tables.
* Only does catalog tables since these are only tables we for sure know
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1417079&r1=1417078&r2=1417079&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Tue Dec 4 18:16:21 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -51,17 +52,15 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.MultithreadedTestUtil;
-import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
import org.apache.hadoop.hbase.MultithreadedTestUtil.RepeatingTestThread;
+import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
@@ -71,8 +70,6 @@ import org.apache.hadoop.hbase.filter.Fi
import org.apache.hadoop.hbase.filter.NullComparator;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
@@ -105,6 +102,7 @@ import com.google.common.collect.Lists;
* HRegions or in the HBaseMaster, so only basic testing is possible.
*/
@Category(MediumTests.class)
+@SuppressWarnings("deprecation")
public class TestHRegion extends HBaseTestCase {
// Do not spin up clusters in here. If you need to spin up a cluster, do it
// over in TestHRegionOnCluster.
@@ -157,7 +155,6 @@ public class TestHRegion extends HBaseTe
String method = "testCompactionAffectedByScanners";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- Configuration conf = HBaseConfiguration.create();
this.region = initHRegion(tableName, method, conf, family);
Put put = new Put(Bytes.toBytes("r1"));
@@ -209,7 +206,6 @@ public class TestHRegion extends HBaseTe
String method = "testToShowNPEOnRegionScannerReseek";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- Configuration conf = HBaseConfiguration.create();
this.region = initHRegion(tableName, method, conf, family);
Put put = new Put(Bytes.toBytes("r1"));
@@ -242,7 +238,6 @@ public class TestHRegion extends HBaseTe
String method = "testSkipRecoveredEditsReplay";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- Configuration conf = HBaseConfiguration.create();
this.region = initHRegion(tableName, method, conf, family);
try {
Path regiondir = region.getRegionDir();
@@ -288,7 +283,7 @@ public class TestHRegion extends HBaseTe
String method = "testSkipRecoveredEditsReplaySomeIgnored";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+ this.region = initHRegion(tableName, method, conf, family);
try {
Path regiondir = region.getRegionDir();
FileSystem fs = region.getFilesystem();
@@ -338,7 +333,7 @@ public class TestHRegion extends HBaseTe
String method = "testSkipRecoveredEditsReplayAllIgnored";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
- this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+ this.region = initHRegion(tableName, method, conf, family);
try {
Path regiondir = region.getRegionDir();
FileSystem fs = region.getFilesystem();
@@ -464,7 +459,7 @@ public class TestHRegion extends HBaseTe
byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
- this.region = initHRegion(TABLE, getName(), FAMILIES);
+ this.region = initHRegion(TABLE, getName(), conf, FAMILIES);
try {
String value = "this is the value";
String value2 = "this is some other value";
@@ -585,7 +580,7 @@ public class TestHRegion extends HBaseTe
public void testFamilyWithAndWithoutColon() throws Exception {
byte [] b = Bytes.toBytes(getName());
byte [] cf = Bytes.toBytes(COLUMN_FAMILY);
- this.region = initHRegion(b, getName(), cf);
+ this.region = initHRegion(b, getName(), conf, cf);
try {
Put p = new Put(b);
byte [] cfwithcolon = Bytes.toBytes(COLUMN_FAMILY + ":");
@@ -609,7 +604,7 @@ public class TestHRegion extends HBaseTe
byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
byte[] qual = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("val");
- this.region = initHRegion(b, getName(), cf);
+ this.region = initHRegion(b, getName(), conf, cf);
try {
HLog.getSyncTime(); // clear counter from prior tests
assertEquals(0, HLog.getSyncTime().count);
@@ -643,7 +638,7 @@ public class TestHRegion extends HBaseTe
Integer lockedRow = region.obtainRowLock(Bytes.toBytes("row_2"));
MultithreadedTestUtil.TestContext ctx =
- new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+ new MultithreadedTestUtil.TestContext(conf);
final AtomicReference<OperationStatus[]> retFromThread =
new AtomicReference<OperationStatus[]>();
TestThread putter = new TestThread(ctx) {
@@ -710,9 +705,7 @@ public class TestHRegion extends HBaseTe
byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
byte[] qual = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("val");
-
- HBaseConfiguration conf = new HBaseConfiguration();
-
+ Configuration conf = HBaseConfiguration.create(this.conf);
// add data with a timestamp that is too recent for range. Ensure assert
conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -759,7 +752,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting empty data in key
Put put = new Put(row1);
@@ -834,7 +827,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in key
Put put = new Put(row1);
@@ -868,7 +861,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in key
Put put = new Put(row1);
@@ -906,7 +899,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in the key to check
Put put = new Put(row1);
@@ -945,7 +938,7 @@ public class TestHRegion extends HBaseTe
}
public void testCheckAndPut_wrongRowInPut() throws IOException {
- this.region = initHRegion(tableName, this.getName(), COLUMNS);
+ this.region = initHRegion(tableName, this.getName(), conf, COLUMNS);
try {
Put put = new Put(row2);
put.add(fam1, qual1, value1);
@@ -980,7 +973,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Put content
Put put = new Put(row1);
@@ -1055,7 +1048,7 @@ public class TestHRegion extends HBaseTe
put.add(fam1, qual, 2, value);
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
region.put(put);
@@ -1085,7 +1078,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1, fam2, fam3);
+ this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3);
try {
List<KeyValue> kvs = new ArrayList<KeyValue>();
kvs.add(new KeyValue(row1, fam4, null, null));
@@ -1123,7 +1116,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("info");
byte [][] families = {fam};
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
@@ -1191,7 +1184,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("info");
byte [][] families = {fam};
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
byte [] row = Bytes.toBytes("table_name");
// column names
@@ -1234,7 +1227,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("info");
byte [][] families = {fam};
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
byte [] row = Bytes.toBytes("row1");
// column names
@@ -1287,7 +1280,7 @@ public class TestHRegion extends HBaseTe
byte[] fam = Bytes.toBytes("info");
byte[][] families = { fam };
String method = this.getName();
- HBaseConfiguration conf = new HBaseConfiguration();
+ Configuration conf = HBaseConfiguration.create(this.conf);
// add data with a timestamp that is too recent for range. Ensure assert
conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
@@ -1318,7 +1311,7 @@ public class TestHRegion extends HBaseTe
byte [] tableName = Bytes.toBytes("test_table");
byte [] fam1 = Bytes.toBytes("columnA");
byte [] fam2 = Bytes.toBytes("columnB");
- this.region = initHRegion(tableName, getName(), fam1, fam2);
+ this.region = initHRegion(tableName, getName(), conf, fam1, fam2);
try {
byte [] rowA = Bytes.toBytes("rowA");
byte [] rowB = Bytes.toBytes("rowB");
@@ -1371,7 +1364,7 @@ public class TestHRegion extends HBaseTe
public void doTestDelete_AndPostInsert(Delete delete)
throws IOException, InterruptedException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
Put put = new Put(row);
@@ -1424,7 +1417,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Building checkerList
List<KeyValue> kvs = new ArrayList<KeyValue>();
@@ -1464,7 +1457,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
Get get = new Get(row1);
get.addColumn(fam2, col1);
@@ -1495,7 +1488,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Add to memstore
Put put = new Put(row1);
@@ -1545,7 +1538,7 @@ public class TestHRegion extends HBaseTe
byte [] fam = Bytes.toBytes("fam");
String method = this.getName();
- this.region = initHRegion(tableName, method, fam);
+ this.region = initHRegion(tableName, method, conf, fam);
try {
Get get = new Get(row);
get.addFamily(fam);
@@ -1565,7 +1558,8 @@ public class TestHRegion extends HBaseTe
public void stestGet_Root() throws IOException {
//Setting up region
String method = this.getName();
- this.region = initHRegion(HConstants.ROOT_TABLE_NAME, method, HConstants.CATALOG_FAMILY);
+ this.region = initHRegion(HConstants.ROOT_TABLE_NAME,
+ method, conf, HConstants.CATALOG_FAMILY);
try {
//Add to memstore
Put put = new Put(HConstants.EMPTY_START_ROW);
@@ -1797,7 +1791,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
Scan scan = new Scan();
scan.addFamily(fam1);
@@ -1822,7 +1816,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
Scan scan = new Scan();
scan.addFamily(fam2);
@@ -1851,7 +1845,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
@@ -1899,7 +1893,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
try {
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
} catch (IOException e) {
e.printStackTrace();
fail("Got IOException during initHRegion, " + e.getMessage());
@@ -1935,7 +1929,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2002,7 +1996,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2062,7 +2056,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2127,7 +2121,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2209,7 +2203,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
//Putting data in Region
Put put = null;
@@ -2270,7 +2264,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in Region
Put put = null;
@@ -2321,7 +2315,7 @@ public class TestHRegion extends HBaseTe
public void testScanner_StopRow1542() throws IOException {
byte [] tableName = Bytes.toBytes("test_table");
byte [] family = Bytes.toBytes("testFamily");
- this.region = initHRegion(tableName, getName(), family);
+ this.region = initHRegion(tableName, getName(), conf, family);
try {
byte [] row1 = Bytes.toBytes("row111");
byte [] row2 = Bytes.toBytes("row222");
@@ -2368,7 +2362,7 @@ public class TestHRegion extends HBaseTe
}
public void testIncrementColumnValue_UpdatingInPlace() throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 1L;
long amount = 3L;
@@ -2396,7 +2390,7 @@ public class TestHRegion extends HBaseTe
public void testIncrementColumnValue_BumpSnapshot() throws IOException {
ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
EnvironmentEdgeManagerTestHelper.injectEdge(mee);
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 42L;
long incr = 44L;
@@ -2435,7 +2429,7 @@ public class TestHRegion extends HBaseTe
}
public void testIncrementColumnValue_ConcurrentFlush() throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 1L;
long amount = 3L;
@@ -2469,7 +2463,7 @@ public class TestHRegion extends HBaseTe
public void testIncrementColumnValue_heapSize() throws IOException {
EnvironmentEdgeManagerTestHelper.injectEdge(new IncrementingEnvironmentEdge());
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long byAmount = 1L;
long size;
@@ -2488,7 +2482,7 @@ public class TestHRegion extends HBaseTe
public void testIncrementColumnValue_UpdatingInPlace_Negative()
throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 3L;
long amount = -1L;
@@ -2509,7 +2503,7 @@ public class TestHRegion extends HBaseTe
public void testIncrementColumnValue_AddingNew()
throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 1L;
long amount = 3L;
@@ -2538,7 +2532,7 @@ public class TestHRegion extends HBaseTe
}
public void testIncrementColumnValue_UpdatingFromSF() throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 1L;
long amount = 3L;
@@ -2566,7 +2560,7 @@ public class TestHRegion extends HBaseTe
public void testIncrementColumnValue_AddingNewAfterSFCheck()
throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 1L;
long amount = 3L;
@@ -2605,7 +2599,7 @@ public class TestHRegion extends HBaseTe
* @throws IOException
*/
public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
long value = 1L;
long amount = 3L;
@@ -2653,7 +2647,7 @@ public class TestHRegion extends HBaseTe
}
public void testIncrementColumnValue_WrongInitialSize() throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
int row1Field1 = 0;
@@ -2681,7 +2675,7 @@ public class TestHRegion extends HBaseTe
}
public void testIncrement_WrongInitialSize() throws IOException {
- this.region = initHRegion(tableName, getName(), fam1);
+ this.region = initHRegion(tableName, getName(), conf, fam1);
try {
byte[] row1 = Bytes.add(Bytes.toBytes("1234"), Bytes.toBytes(0L));
long row1Field1 = 0;
@@ -2757,7 +2751,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = this.getName();
- this.region = initHRegion(tableName, method, fam1);
+ this.region = initHRegion(tableName, method, conf, fam1);
try {
//Putting data in Region
KeyValue kv14 = new KeyValue(row1, fam1, qf1, ts4, KeyValue.Type.Put, null);
@@ -2972,7 +2966,7 @@ public class TestHRegion extends HBaseTe
int compactInterval = 10 * flushAndScanInterval;
String method = "testFlushCacheWhileScanning";
- this.region = initHRegion(tableName,method, family);
+ this.region = initHRegion(tableName,method, conf, family);
try {
FlushThread flushThread = new FlushThread();
flushThread.start();
@@ -3103,7 +3097,7 @@ public class TestHRegion extends HBaseTe
}
String method = "testWritesWhileScanning";
- this.region = initHRegion(tableName, method, families);
+ this.region = initHRegion(tableName, method, conf, families);
try {
PutThread putThread = new PutThread(numRows, families, qualifiers);
putThread.start();
@@ -3225,6 +3219,8 @@ public class TestHRegion extends HBaseTe
}
numPutsFinished++;
}
+ } catch (InterruptedIOException e) {
+ // This is fine. It means we are done, or didn't get the lock on time
} catch (IOException e) {
LOG.error("error while putting records", e);
error = e;
@@ -3261,8 +3257,9 @@ public class TestHRegion extends HBaseTe
qualifiers[i] = Bytes.toBytes("qual" + i);
}
+ Configuration conf = HBaseConfiguration.create(this.conf);
+
String method = "testWritesWhileGetting";
- Configuration conf = HBaseConfiguration.create();
// This test flushes constantly and can cause many files to be created, possibly
// extending over the ulimit. Make sure compactions are aggressive in reducing
// the number of HFiles created.
@@ -3271,7 +3268,7 @@ public class TestHRegion extends HBaseTe
this.region = initHRegion(tableName, method, conf, families);
PutThread putThread = null;
MultithreadedTestUtil.TestContext ctx =
- new MultithreadedTestUtil.TestContext(HBaseConfiguration.create());
+ new MultithreadedTestUtil.TestContext(conf);
try {
putThread = new PutThread(numRows, families, qualifiers);
putThread.start();
@@ -3357,7 +3354,7 @@ public class TestHRegion extends HBaseTe
byte[] tableName = Bytes.toBytes(method);
byte[] family = Bytes.toBytes("family");
this.region = initHRegion(tableName, Bytes.toBytes("x"), Bytes.toBytes("z"), method,
- HBaseConfiguration.create(), family);
+ conf, family);
try {
byte[] rowNotServed = Bytes.toBytes("a");
Get g = new Get(rowNotServed);
@@ -3421,7 +3418,7 @@ public class TestHRegion extends HBaseTe
//Setting up region
String method = "testIndexesScanWithOneDeletedRow";
- this.region = initHRegion(tableName, method, HBaseConfiguration.create(), family);
+ this.region = initHRegion(tableName, method, conf, family);
try {
Put put = new Put(Bytes.toBytes(1L));
put.add(family, qual1, 1L, Bytes.toBytes(1L));
@@ -3874,7 +3871,6 @@ public class TestHRegion extends HBaseTe
*/
@Test
public void testParallelIncrementWithMemStoreFlush() throws Exception {
- Configuration conf = HBaseConfiguration.create();
String method = "testParallelIncrementWithMemStoreFlush";
byte[] tableName = Bytes.toBytes(method);
byte[] family = Incrementer.family;
@@ -4007,7 +4003,8 @@ public class TestHRegion extends HBaseTe
}
private Configuration initSplit() {
- Configuration conf = HBaseConfiguration.create();
+ Configuration conf = HBaseConfiguration.create(this.conf);
+
// Always compact if there is more than one store file.
conf.setInt("hbase.hstore.compactionThreshold", 2);
@@ -4028,19 +4025,6 @@ public class TestHRegion extends HBaseTe
/**
* @param tableName
* @param callingMethod
- * @param families
- * @return A region on which you must call {@link HRegion#closeHRegion(HRegion)} when done.
- * @throws IOException
- */
- private static HRegion initHRegion (byte [] tableName, String callingMethod,
- byte[] ... families)
- throws IOException {
- return initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
- }
-
- /**
- * @param tableName
- * @param callingMethod
* @param conf
* @param families
* @throws IOException
Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java?rev=1417079&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java (added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java Tue Dec 4 18:16:21 2012
@@ -0,0 +1,90 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.RegionTooBusyException;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * TestHRegion with hbase.busy.wait.duration set to 1000 (1 second).
+ * We can't use parameterized test since TestHRegion is old fashion.
+ */
+@Category(MediumTests.class)
+@SuppressWarnings("deprecation")
+public class TestHRegionBusyWait extends TestHRegion {
+ public TestHRegionBusyWait() {
+ conf.set("hbase.busy.wait.duration", "1000");
+ }
+
+ /**
+ * Test RegionTooBusyException thrown when region is busy
+ */
+ @Test (timeout=2000)
+ public void testRegionTooBusy() throws IOException {
+ String method = "testRegionTooBusy";
+ byte[] tableName = Bytes.toBytes(method);
+ byte[] family = Bytes.toBytes("family");
+ region = initHRegion(tableName, method, conf, family);
+ final AtomicBoolean stopped = new AtomicBoolean(true);
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ region.lock.writeLock().lock();
+ stopped.set(false);
+ while (!stopped.get()) {
+ Thread.sleep(100);
+ }
+ } catch (InterruptedException ie) {
+ } finally {
+ region.lock.writeLock().unlock();
+ }
+ }
+ });
+ t.start();
+ Get get = new Get(row);
+ try {
+ while (stopped.get()) {
+ Thread.sleep(100);
+ }
+ region.get(get, null);
+ fail("Should throw RegionTooBusyException");
+ } catch (InterruptedException ie) {
+ fail("test interrupted");
+ } catch (RegionTooBusyException e) {
+ // Good, expected
+ } finally {
+ stopped.set(true);
+ try {
+ t.join();
+ } catch (Throwable e) {
+ }
+
+ HRegion.closeHRegion(region);
+ region = null;
+ }
+ }
+}
Propchange: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionBusyWait.java
------------------------------------------------------------------------------
svn:eol-style = native