You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2008/11/28 11:01:39 UTC
svn commit: r721410 - in /hadoop/hbase/trunk/src:
java/org/apache/hadoop/hbase/client/ java/org/apache/hadoop/hbase/master/
java/org/apache/hadoop/hbase/regionserver/
test/org/apache/hadoop/hbase/client/
Author: apurtell
Date: Fri Nov 28 02:01:39 2008
New Revision: 721410
URL: http://svn.apache.org/viewvc?rev=721410&view=rev
Log:
HBASE-902 Add force compaction and force split operations to UI and Admin; force compactions before split, added test case
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java (with props)
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Fri Nov 28 02:01:39 2008
@@ -529,6 +529,7 @@
arr[0] = (HTableDescriptor)args[0];
this.master.modifyTable(tableName, op, arr);
} break;
+ case HConstants.MODIFY_TABLE_COMPACT:
case HConstants.MODIFY_TABLE_SPLIT: {
Writable[] arr = null;
if (args != null && args.length > 0) {
@@ -538,8 +539,7 @@
} else if (args[0] instanceof ImmutableBytesWritable) {
arr[0] = (ImmutableBytesWritable)args[0];
} else {
- throw new IOException(
- "SPLIT with arg requires byte[] or ImmutableBytesWritable");
+ throw new IOException("SPLIT or COMPACT with arg requires byte[] or ImmutableBytesWritable");
}
}
this.master.modifyTable(tableName, op, arr);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java Fri Nov 28 02:01:39 2008
@@ -996,6 +996,10 @@
while (i.hasNext()) {
Pair<HRegionInfo,HServerAddress> pair = i.next();
if (addr.equals(pair.getSecond())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("sending MSG_REGION_COMPACT " + pair.getFirst() + " to " +
+ addr);
+ }
returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_COMPACT, pair.getFirst()));
i.remove();
}
@@ -1004,6 +1008,10 @@
while (i.hasNext()) {
Pair<HRegionInfo,HServerAddress> pair = i.next();
if (addr.equals(pair.getSecond())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("sending MSG_REGION_SPLIT " + pair.getFirst() + " to " +
+ addr);
+ }
returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_SPLIT, pair.getFirst()));
i.remove();
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Fri Nov 28 02:01:39 2008
@@ -534,8 +534,7 @@
byte [] startKey = this.regionInfo.getStartKey();
byte [] endKey = this.regionInfo.getEndKey();
if (HStoreKey.equalsTwoRowKeys(this.regionInfo,startKey, midKey)) {
- LOG.debug("Startkey (" + startKey + ") and midkey + (" +
- midKey + ") are same, not splitting");
+ LOG.debug("Startkey and midkey are same, not splitting");
return null;
}
if (HStoreKey.equalsTwoRowKeys(this.regionInfo,midKey, endKey)) {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri Nov 28 02:01:39 2008
@@ -1018,7 +1018,11 @@
case MSG_REGION_SPLIT: {
// Force split a region
HRegion region = getRegion(info.getRegionName());
+ // flush the memcache for the region
+ region.flushcache();
+ // flag that the region should be split
region.regionInfo.shouldSplit(true);
+ // force a compaction
compactSplitThread.compactionRequested(region,
"MSG_REGION_SPLIT");
} break;
@@ -1026,6 +1030,9 @@
case MSG_REGION_COMPACT: {
// Compact a region
HRegion region = getRegion(info.getRegionName());
+ // flush the memcache for the region
+ region.flushcache();
+ // force a compaction
compactSplitThread.compactionRequested(region,
"MSG_REGION_COMPACT");
} break;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Fri Nov 28 02:01:39 2008
@@ -823,6 +823,7 @@
List<HStoreFile> filesToCompact = null;
synchronized (storefiles) {
if (this.storefiles.size() <= 0) {
+ LOG.debug("no store files to compact");
return null;
}
// filesToCompact are sorted oldest to newest.
@@ -839,8 +840,8 @@
doMajorCompaction = isMajorCompaction(filesToCompact);
}
boolean references = hasReferences(filesToCompact);
- if (!doMajorCompaction && !references &&
- filesToCompact.size() < compactionThreshold) {
+ if (!doMajorCompaction && !references &&
+ (forceSplit || (filesToCompact.size() < compactionThreshold))) {
return checkSplit(forceSplit);
}
if (!fs.exists(compactionDir) && !fs.mkdirs(compactionDir)) {
@@ -2036,6 +2037,9 @@
splitable = !curHSF.isReference();
if (!splitable) {
// RETURN IN MIDDLE OF FUNCTION!!! If not splitable, just return.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(curHSF + " is not splittable");
+ }
return null;
}
}
@@ -2063,6 +2067,9 @@
// (ever) split this region.
if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(), firstKey.getRow()) &&
HStoreKey.equalsTwoRowKeys(info, mk.getRow(), lastKey.getRow())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("cannot split because midkey is the same as first or last row");
+ }
return null;
}
return new StoreSize(maxSize, mk.getRow());
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java?rev=721410&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java Fri Nov 28 02:01:39 2008
@@ -0,0 +1,119 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Tests forced splitting of HTable
+ */
+public class TestForceSplit extends HBaseClusterTestCase {
+ private static final byte[] tableName = Bytes.toBytes("test");
+ private static final byte[] columnName = Bytes.toBytes("a:");
+ private static final byte[] key_mmi = Bytes.toBytes("mmi");
+ private static final byte[] key_ssm = Bytes.toBytes("ssm");
+
+ /**
+ * the test
+ * @throws IOException
+ */
+ public void testHTable() throws Exception {
+ // create the test table
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(columnName));
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ admin.createTable(htd);
+ HTable table = new HTable(conf, tableName);
+ byte[] k = new byte[3];
+ for (byte b1 = 'a'; b1 < 'z'; b1++) {
+ for (byte b2 = 'a'; b2 < 'z'; b2++) {
+ for (byte b3 = 'a'; b3 < 'z'; b3++) {
+ k[0] = b1;
+ k[1] = b2;
+ k[2] = b3;
+ BatchUpdate update = new BatchUpdate(k);
+ update.put(columnName, k);
+ table.commit(update);
+ }
+ }
+ }
+
+ // get the initial layout (should just be one region)
+ Map<HRegionInfo,HServerAddress> m = table.getRegionsInfo();
+ System.out.println("Initial regions (" + m.size() + "): " + m);
+ assertTrue(m.size() == 1);
+
+ // tell the master to split the table
+ admin.modifyTable(tableName, HConstants.MODIFY_TABLE_SPLIT);
+
+ // give some time for the split to happen
+ Thread.sleep(15 * 1000);
+
+ // check again
+ table = new HTable(conf, tableName);
+ m = table.getRegionsInfo();
+ System.out.println("Regions after split (" + m.size() + "): " + m);
+ // should have two regions now
+ assertTrue(m.size() == 2);
+ // and "mmi" should be the midpoint
+ for (HRegionInfo hri: m.keySet()) {
+ byte[] start = hri.getStartKey();
+ byte[] end = hri.getEndKey();
+ if (Bytes.equals(start, HConstants.EMPTY_BYTE_ARRAY))
+ assertTrue(Bytes.equals(end, key_mmi));
+ if (Bytes.equals(end, key_mmi))
+ assertTrue(Bytes.equals(start, HConstants.EMPTY_BYTE_ARRAY));
+ }
+
+ // tell the master to split the table again, the second half
+ admin.modifyTable(tableName, HConstants.MODIFY_TABLE_SPLIT, key_mmi);
+
+ // give some time for the split to happen
+ Thread.sleep(15 * 1000);
+
+ // check again
+ table = new HTable(conf, tableName);
+ m = table.getRegionsInfo();
+ System.out.println("Regions after split (" + m.size() + "): " + m);
+ // should have three regions now
+ assertTrue(m.size() == 3);
+ // and "mmi" and "ssm" should be the midpoints
+ for (HRegionInfo hri: m.keySet()) {
+ byte[] start = hri.getStartKey();
+ byte[] end = hri.getEndKey();
+ if (Bytes.equals(start, HConstants.EMPTY_BYTE_ARRAY))
+ assertTrue(Bytes.equals(end, key_mmi));
+ if (Bytes.equals(start, key_mmi))
+ assertTrue(Bytes.equals(end, key_ssm));
+ if (Bytes.equals(start, key_ssm))
+ assertTrue(Bytes.equals(end, HConstants.EMPTY_BYTE_ARRAY));
+ }
+ }
+}
Propchange: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java
------------------------------------------------------------------------------
svn:executable = *