You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/12/08 04:01:25 UTC
svn commit: r1548981 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
Author: liyin
Date: Sun Dec 8 03:01:24 2013
New Revision: 1548981
URL: http://svn.apache.org/r1548981
Log:
[HBASE-8805] Add online configeration supporting and corresponding testcases
Summary: HRegion implements ConfigurationObserver, register/deregister from HRegionServer.configurationManager, testcases
Test Plan: Run testcase
Reviewers: gauravm, zelaine, liyintang
Reviewed By: liyintang
CC: mahesh, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1067616
Task ID: 2977299
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1548981&r1=1548980&r2=1548981&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sun Dec 8 03:01:24 2013
@@ -26,13 +26,11 @@ import java.io.UnsupportedEncodingExcept
import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -55,6 +53,8 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.Date;
+import java.text.SimpleDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -72,15 +72,14 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RowLock;
-import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.Reference.Range;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -144,7 +143,7 @@ import com.google.common.collect.Lists;
* regionName is a unique identifier for this HRegion. (startKey, endKey]
* defines the keyspace for this HRegion.
*/
-public class HRegion implements HeapSize, ConfigurationObserver {
+public class HRegion implements HeapSize {
public static final Log LOG = LogFactory.getLog(HRegion.class);
static final String SPLITDIR = "splits";
static final String MERGEDIR = "merges";
@@ -586,8 +585,6 @@ public class HRegion implements HeapSize
*/
public long initialize(final Progressable reporter)
throws IOException {
- HRegionServer.configurationManager.registerObserver(this);
-
MonitoredTask status = TaskMonitor.get().createStatus(
"Initializing region " + this);
try {
@@ -838,8 +835,6 @@ public class HRegion implements HeapSize
* @throws IOException e
*/
public List<StoreFile> close(final boolean abort) throws IOException {
- HRegionServer.configurationManager.deregisterObserver(this);
-
MonitoredTask status = TaskMonitor.get().createStatus(
"Closing region " + this + (abort ? " due to abort" : ""));
if (isClosed()) {
@@ -4129,4 +4124,5 @@ public class HRegion implements HeapSize
s.updateConfiguration();
}
}
+
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java?rev=1548981&r1=1548980&r2=1548981&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/TestPerColumnFamilyFlush.java Sun Dec 8 03:01:24 2013
@@ -20,13 +20,8 @@
package org.apache.hadoop.hbase;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
import junit.framework.Assert;
import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -41,6 +36,10 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.junit.Test;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
/**
* This test verifies the correctness of the Per Column Family flushing strategy
*/
@@ -61,8 +60,9 @@ public class TestPerColumnFamilyFlush ex
public static final byte[] FAMILY2 = families[1];
public static final byte[] FAMILY3 = families[2];
- private void initHRegion (String callingMethod, Configuration conf)
- throws IOException {
+ private void initHRegion (String callingMethod,
+ HBaseConfiguration conf)
+ throws IOException {
HTableDescriptor htd = new HTableDescriptor(TABLENAME);
for(byte [] family : families) {
htd.addFamily(new HColumnDescriptor(family));
@@ -105,12 +105,12 @@ public class TestPerColumnFamilyFlush ex
@Test
public void testSelectiveFlushWhenEnabled() throws IOException {
// Set up the configuration
- Configuration conf = HBaseConfiguration.create();
+ HBaseConfiguration conf = new HBaseConfiguration();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200*1024);
conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, true);
conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, 100*1024);
- // Initialize the HRegion
+ // Intialize the HRegion
initHRegion(getName(), conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {
@@ -247,123 +247,15 @@ public class TestPerColumnFamilyFlush ex
Assert.assertEquals(0, region.getMemstoreSize().get());
}
- public void testSelectiveFlushWithThreshold(long t) throws IOException {
- /* t->|
- * Phase 1 | | |
- * +-----+-----+
- * FM1 FM2 FM3
- */
- // flush all
- region.flushcache(false);
- region.put(createPut(2, 1));
- region.put(createPut(3, 1));
-
-
- int cnt = 0;
- while (region.getStore(FAMILY1).getMemStoreSize() <= t) {
- region.put(createPut(1, cnt));
- cnt ++;
- }
-
- final long sizeFamily1_1 = region.getStore(FAMILY1).getMemStoreSize();
- final long sizeFamily2_1 = region.getStore(FAMILY2).getMemStoreSize();
- final long sizeFamily3_1 = region.getStore(FAMILY3).getMemStoreSize();
-
- Assert.assertTrue(sizeFamily1_1 > t);
- Assert.assertTrue(sizeFamily2_1 > 0);
- Assert.assertTrue(sizeFamily3_1 > 0);
-
- /* t->
- * Phase 2 | |
- * +-----+-----+
- * FM1 FM2 FM3
- */
- // flush, should only flush family 1
- region.flushcache(true);
-
- final long sizeFamily1_2 = region.getStore(FAMILY1).getMemStoreSize();
- final long sizeFamily2_2 = region.getStore(FAMILY2).getMemStoreSize();
- final long sizeFamily3_2 = region.getStore(FAMILY3).getMemStoreSize();
-
- // should clear FAMILY1 only
- Assert.assertEquals("sizeFamily1", 0, sizeFamily1_2);
- Assert.assertEquals("sizeFamily2", sizeFamily2_1, sizeFamily2_2);
- Assert.assertEquals("sizeFamily3", sizeFamily3_1, sizeFamily3_2);
- /* t->,
- * Phase 3 | | |
- * +-----+-----+
- * FM1 FM2 FM3
- */
- // flush all
- region.flushcache(false);
-
- region.put(createPut(2, 1));
- region.put(createPut(3, 1));
- // reduce cnt to 1 to make the size less than t
- cnt--;
- for (int i = 0; i < cnt; i ++) {
- region.put(createPut(1, i));
- }
-
- final long sizeFamily1_3 = region.getStore(FAMILY1).getMemStoreSize();
- final long sizeFamily2_3 = region.getStore(FAMILY2).getMemStoreSize();
- final long sizeFamily3_3 = region.getStore(FAMILY3).getMemStoreSize();
-
- Assert.assertTrue("sizeFamily1 < t", sizeFamily1_3 < t);
- Assert.assertEquals("sizeFamily2", sizeFamily2_1, sizeFamily2_3);
- Assert.assertEquals("sizeFamily3", sizeFamily3_1, sizeFamily3_3);
- /* t->
- * Phase 3
- * +-----+-----+
- * FM1 FM2 FM3
- */
- // flush, should flush all
- region.flushcache(true);
-
- final long sizeFamily1_4 = region.getStore(FAMILY1).getMemStoreSize();
- final long sizeFamily2_4 = region.getStore(FAMILY2).getMemStoreSize();
- final long sizeFamily3_4 = region.getStore(FAMILY3).getMemStoreSize();
- Assert.assertEquals("sizeFamily1", 0, sizeFamily1_4);
- Assert.assertEquals("sizeFamily2", 0, sizeFamily2_4);
- Assert.assertEquals("sizeFamily3", 0, sizeFamily3_4);
- }
-
- @Test
- public void testConfigueChange() throws IOException {
- final int T_init = 100*1024;
- final int T_1 = 200*1024;
- final int T_2 = 50*1024;
-
- // Set up the configuration
- Configuration conf = HBaseConfiguration.create();
- conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200*1024);
- conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, true);
- conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_init);
-
- // Initialize the HRegion
- initHRegion(getName(), conf);
-
- // test for threshold of T_init
- this.testSelectiveFlushWithThreshold(T_init);
- // test for threshold of T_1
- conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_1);
- HRegionServer.configurationManager.notifyAllObservers(conf);
- this.testSelectiveFlushWithThreshold(T_1);
- // test for threshold of T_2
- conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, T_2);
- HRegionServer.configurationManager.notifyAllObservers(conf);
- this.testSelectiveFlushWithThreshold(T_2);
- }
-
@Test
public void testSelectiveFlushWhenNotEnabled() throws IOException {
// Set up the configuration
- Configuration conf = HBaseConfiguration.create();
+ HBaseConfiguration conf = new HBaseConfiguration();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 200 * 1024);
conf.setBoolean(HConstants.HREGION_MEMSTORE_PER_COLUMN_FAMILY_FLUSH, false);
conf.setLong(HConstants.HREGION_MEMSTORE_COLUMNFAMILY_FLUSH_SIZE, 100 * 1024);
- // Initialize the HRegion
+ // Intialize the HRegion
initHRegion(getName(), conf);
// Add 1200 entries for CF1, 100 for CF2 and 50 for CF3
for (int i = 1; i <= 1200; i++) {